doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -5,7 +5,9 @@ import {
|
|
|
5
5
|
recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
|
|
6
6
|
runSiLURowSplit, recordSiLURowSplit,
|
|
7
7
|
runMatmulRMSNormFused, recordMatmulRMSNormFused,
|
|
8
|
+
runRMSNormStats, recordRMSNormStats,
|
|
8
9
|
runSandwichRMSNormPair, recordSandwichRMSNormPair,
|
|
10
|
+
runResidualNextRMSNormPair, recordResidualNextRMSNormPair,
|
|
9
11
|
runConv2D, recordConv2D,
|
|
10
12
|
} from '../../../gpu/kernel-selector.js';
|
|
11
13
|
import {
|
|
@@ -16,7 +18,7 @@ import {
|
|
|
16
18
|
} from '../../../gpu/kernels/cast.js';
|
|
17
19
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
18
20
|
import { releaseBuffer, readBuffer, acquireBuffer, uploadData } from '../../../memory/buffer-pool.js';
|
|
19
|
-
import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
21
|
+
import { getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
20
22
|
import { kernelTrace, traceStep } from './kernel-trace.js';
|
|
21
23
|
import {
|
|
22
24
|
runLayerAttentionGPU,
|
|
@@ -58,6 +60,12 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
|
|
|
58
60
|
return result;
|
|
59
61
|
}
|
|
60
62
|
|
|
63
|
+
export async function doRMSNormStats(input, residual, eps, options, recorder) {
|
|
64
|
+
return recorder
|
|
65
|
+
? await recordRMSNormStats(recorder, input, residual, eps, options)
|
|
66
|
+
: await runRMSNormStats(input, residual, eps, options);
|
|
67
|
+
}
|
|
68
|
+
|
|
61
69
|
export async function doSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options, recorder) {
|
|
62
70
|
const result = recorder
|
|
63
71
|
? await recordSandwichRMSNormPair(recorder, input, residual, postWeight, preWeight, eps, options)
|
|
@@ -73,6 +81,20 @@ export async function doSandwichRMSNormPair(input, residual, postWeight, preWeig
|
|
|
73
81
|
return result;
|
|
74
82
|
}
|
|
75
83
|
|
|
84
|
+
export async function doResidualNextRMSNormPair(input, residual, normWeight, eps, options = {}, recorder) {
|
|
85
|
+
const result = recorder
|
|
86
|
+
? await recordResidualNextRMSNormPair(recorder, input, residual, normWeight, eps, options)
|
|
87
|
+
: await runResidualNextRMSNormPair(input, residual, normWeight, eps, options);
|
|
88
|
+
|
|
89
|
+
if (kernelTrace.enabled && !recorder) {
|
|
90
|
+
const layer = options.layerIdx ?? -1;
|
|
91
|
+
const label = options.label ?? 'residual_rmsnorm_pair';
|
|
92
|
+
await traceStep('residual_rmsnorm_pair.residual', label, layer, result.residual.buffer, [options.batchSize, options.hiddenSize]);
|
|
93
|
+
await traceStep('residual_rmsnorm_pair.next_input', label, layer, result.nextInput.buffer, [options.batchSize, options.hiddenSize]);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return result;
|
|
97
|
+
}
|
|
76
98
|
|
|
77
99
|
export async function doResidualAdd(a, b, size, recorder, traceOptions) {
|
|
78
100
|
const options = {};
|
|
@@ -180,6 +202,17 @@ export async function doMatmulRMSNormFused(input, weight, normWeight, options, r
|
|
|
180
202
|
return resultTensor;
|
|
181
203
|
}
|
|
182
204
|
|
|
205
|
+
function requireProjectionWeightDtype(weight, declared, label) {
|
|
206
|
+
if (isWeightBuffer(weight)) {
|
|
207
|
+
return getWeightDtype(weight);
|
|
208
|
+
}
|
|
209
|
+
const dtype = declared ?? weight?.dtype;
|
|
210
|
+
if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') {
|
|
211
|
+
return dtype;
|
|
212
|
+
}
|
|
213
|
+
throw new Error(`${label} requires explicit weight dtype.`);
|
|
214
|
+
}
|
|
215
|
+
|
|
183
216
|
export async function doConv(
|
|
184
217
|
inputTensor,
|
|
185
218
|
convInProj,
|
|
@@ -200,6 +233,16 @@ export async function doConv(
|
|
|
200
233
|
if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
|
|
201
234
|
throw new Error('doConv requires hiddenSize > 0.');
|
|
202
235
|
}
|
|
236
|
+
const convInProjDtype = requireProjectionWeightDtype(
|
|
237
|
+
convInProj,
|
|
238
|
+
options.convInProjDtype ?? options.weightDtype,
|
|
239
|
+
`${label}.in_proj`
|
|
240
|
+
);
|
|
241
|
+
const convOutProjDtype = requireProjectionWeightDtype(
|
|
242
|
+
convOutProj,
|
|
243
|
+
options.convOutProjDtype ?? options.weightDtype,
|
|
244
|
+
`${label}.out_proj`
|
|
245
|
+
);
|
|
203
246
|
|
|
204
247
|
// LFM2 gated short convolution (GPU-native):
|
|
205
248
|
// in_proj → 3×hidden → GPU kernel: split(B,C,x) + B*x + causal conv1d + C*conv_out → out_proj
|
|
@@ -225,6 +268,8 @@ export async function doConv(
|
|
|
225
268
|
kernelPath,
|
|
226
269
|
role: 'conv_in_proj',
|
|
227
270
|
executionPolicies: options.executionPolicies ?? null,
|
|
271
|
+
bDtype: convInProjDtype,
|
|
272
|
+
outputDtype: inputTensor.dtype,
|
|
228
273
|
},
|
|
229
274
|
recorder
|
|
230
275
|
);
|
|
@@ -265,6 +310,8 @@ export async function doConv(
|
|
|
265
310
|
kernelPath,
|
|
266
311
|
role: 'conv_out_proj',
|
|
267
312
|
executionPolicies: options.executionPolicies ?? null,
|
|
313
|
+
bDtype: convOutProjDtype,
|
|
314
|
+
outputDtype: convOut.dtype,
|
|
268
315
|
},
|
|
269
316
|
recorder
|
|
270
317
|
);
|
|
@@ -25,12 +25,26 @@ import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRang
|
|
|
25
25
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
26
26
|
import { getDevice } from '../../../gpu/device.js';
|
|
27
27
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
28
|
+
import { f16ToF32 } from '../../../loader/dtype-utils.js';
|
|
28
29
|
|
|
29
30
|
const pleRuntimeCache = new WeakMap();
|
|
30
31
|
const pleRangeRowCache = new WeakMap();
|
|
32
|
+
let f16ToF32Lookup = null;
|
|
31
33
|
const PLE_ACTIVATION_STATIC_QMIN = -128;
|
|
32
34
|
const PLE_ACTIVATION_STATIC_QMAX = 127;
|
|
33
35
|
|
|
36
|
+
function getF16ToF32Lookup() {
|
|
37
|
+
if (f16ToF32Lookup) {
|
|
38
|
+
return f16ToF32Lookup;
|
|
39
|
+
}
|
|
40
|
+
const lookup = new Float32Array(1 << 16);
|
|
41
|
+
for (let value = 0; value < lookup.length; value += 1) {
|
|
42
|
+
lookup[value] = f16ToF32(value);
|
|
43
|
+
}
|
|
44
|
+
f16ToF32Lookup = lookup;
|
|
45
|
+
return lookup;
|
|
46
|
+
}
|
|
47
|
+
|
|
34
48
|
function getPerLayerInputWeights(context) {
|
|
35
49
|
const weights = context.weights.get('per_layer_inputs');
|
|
36
50
|
if (!weights || typeof weights !== 'object') {
|
|
@@ -431,7 +445,7 @@ export async function ensurePleScaledProjectionNormWeight(context, combineScale
|
|
|
431
445
|
}
|
|
432
446
|
scaledProjectionNormWeight = createPleProjectionNormTensor(
|
|
433
447
|
scaledBuffer,
|
|
434
|
-
|
|
448
|
+
inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
|
|
435
449
|
hiddenSizePerLayerInput,
|
|
436
450
|
'per_layer_projection_norm_scaled'
|
|
437
451
|
);
|
|
@@ -474,6 +488,17 @@ function getEmbeddingDtype(weight) {
|
|
|
474
488
|
return getWeightDtype(weight);
|
|
475
489
|
}
|
|
476
490
|
|
|
491
|
+
function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
|
|
492
|
+
if (rawDtype == null) {
|
|
493
|
+
throw new Error(`${label} requires source dtype metadata.`);
|
|
494
|
+
}
|
|
495
|
+
const sourceDtype = String(rawDtype).toLowerCase();
|
|
496
|
+
if (!allowed.includes(sourceDtype)) {
|
|
497
|
+
throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
|
|
498
|
+
}
|
|
499
|
+
return sourceDtype;
|
|
500
|
+
}
|
|
501
|
+
|
|
477
502
|
function getEmbeddingTranspose(weight) {
|
|
478
503
|
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
|
|
479
504
|
return weight.layout === 'column';
|
|
@@ -614,6 +639,28 @@ function getPleHotCachePolicy(sessionConfig) {
|
|
|
614
639
|
return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
|
|
615
640
|
}
|
|
616
641
|
|
|
642
|
+
export function resolvePleHotVocabularyCapacity({
|
|
643
|
+
maxTokens,
|
|
644
|
+
maxBytes,
|
|
645
|
+
numLayers,
|
|
646
|
+
hiddenSize,
|
|
647
|
+
bytesPerElement,
|
|
648
|
+
vocabSize,
|
|
649
|
+
}) {
|
|
650
|
+
const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
|
|
651
|
+
const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
|
|
652
|
+
const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
|
|
653
|
+
const maxTableRows = bytesPerHotRow > 0
|
|
654
|
+
? Math.floor(tableBudgetBytes / bytesPerHotRow)
|
|
655
|
+
: 0;
|
|
656
|
+
const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
|
|
657
|
+
return {
|
|
658
|
+
maxHotTokens,
|
|
659
|
+
bytesPerHotRow,
|
|
660
|
+
tokenIndexMapBytes,
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
|
|
617
664
|
export function getPleHotVocabularyRuntime(context) {
|
|
618
665
|
const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
|
|
619
666
|
if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
|
|
@@ -721,11 +768,12 @@ function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activat
|
|
|
721
768
|
}
|
|
722
769
|
|
|
723
770
|
function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
|
|
724
|
-
const sourceDtype =
|
|
771
|
+
const sourceDtype = requirePleSourceDtype(
|
|
725
772
|
(isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
|
|
726
|
-
?? embedTokensPerLayer?.dtype
|
|
727
|
-
|
|
728
|
-
|
|
773
|
+
?? embedTokensPerLayer?.dtype,
|
|
774
|
+
'Gemma 4 range-backed per-layer input rows',
|
|
775
|
+
['f16', 'bf16', 'f32']
|
|
776
|
+
);
|
|
729
777
|
const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
|
|
730
778
|
return {
|
|
731
779
|
sourceDtype,
|
|
@@ -916,12 +964,10 @@ export async function ensurePleGpuSplitTablesRuntime(context) {
|
|
|
916
964
|
throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
|
|
917
965
|
}
|
|
918
966
|
|
|
919
|
-
const sourceDtype =
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
);
|
|
924
|
-
}
|
|
967
|
+
const sourceDtype = requirePleSourceDtype(
|
|
968
|
+
embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
|
|
969
|
+
'Gemma 4 gpu_split_tables materialization'
|
|
970
|
+
);
|
|
925
971
|
|
|
926
972
|
const device = getDevice();
|
|
927
973
|
if (!device) {
|
|
@@ -1085,19 +1131,32 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1085
1131
|
return null;
|
|
1086
1132
|
}
|
|
1087
1133
|
|
|
1134
|
+
const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
|
|
1135
|
+
const capacity = resolvePleHotVocabularyCapacity({
|
|
1136
|
+
maxTokens: policy.maxTokens,
|
|
1137
|
+
maxBytes: policy.maxBytes,
|
|
1138
|
+
numLayers,
|
|
1139
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1140
|
+
bytesPerElement,
|
|
1141
|
+
vocabSize: vocabSizePerLayerInput,
|
|
1142
|
+
});
|
|
1143
|
+
if (capacity.maxHotTokens <= 0) {
|
|
1144
|
+
return null;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1088
1147
|
const tokenizer = context?.tokenizer ?? null;
|
|
1089
1148
|
const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
|
|
1090
|
-
? tokenizer.getHotTokenIds(
|
|
1149
|
+
? tokenizer.getHotTokenIds(capacity.maxHotTokens)
|
|
1091
1150
|
: null;
|
|
1092
1151
|
const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
|
|
1093
1152
|
context,
|
|
1094
|
-
|
|
1153
|
+
capacity.maxHotTokens,
|
|
1095
1154
|
vocabSizePerLayerInput
|
|
1096
1155
|
);
|
|
1097
1156
|
const hotTokenIds = mergePleHotVocabularyTokenIds(
|
|
1098
1157
|
seedTokenIds,
|
|
1099
1158
|
tokenizerHotTokenIds,
|
|
1100
|
-
|
|
1159
|
+
capacity.maxHotTokens,
|
|
1101
1160
|
vocabSizePerLayerInput
|
|
1102
1161
|
);
|
|
1103
1162
|
if (hotTokenIds.length === 0) {
|
|
@@ -1109,7 +1168,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1109
1168
|
const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
|
|
1110
1169
|
if (
|
|
1111
1170
|
cached
|
|
1112
|
-
&& cached.maxTokens ===
|
|
1171
|
+
&& cached.maxTokens === capacity.maxHotTokens
|
|
1172
|
+
&& cached.maxBytes === policy.maxBytes
|
|
1113
1173
|
&& cached.outputDtype === policy.outputDtype
|
|
1114
1174
|
&& cached.vocabSize === vocabSizePerLayerInput
|
|
1115
1175
|
&& cached.numLayers === numLayers
|
|
@@ -1124,11 +1184,15 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1124
1184
|
return null;
|
|
1125
1185
|
}
|
|
1126
1186
|
|
|
1127
|
-
const sourceDtype =
|
|
1128
|
-
|
|
1187
|
+
const sourceDtype = requirePleSourceDtype(
|
|
1188
|
+
embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
|
|
1189
|
+
'Gemma 4 hot vocabulary cache'
|
|
1190
|
+
);
|
|
1191
|
+
const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
|
|
1192
|
+
if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
|
|
1129
1193
|
throw new Error(
|
|
1130
|
-
`Gemma 4 hot vocabulary cache
|
|
1131
|
-
`
|
|
1194
|
+
`Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
|
|
1195
|
+
`to output dtype "${policy.outputDtype}".`
|
|
1132
1196
|
);
|
|
1133
1197
|
}
|
|
1134
1198
|
|
|
@@ -1137,7 +1201,6 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1137
1201
|
throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
|
|
1138
1202
|
}
|
|
1139
1203
|
|
|
1140
|
-
const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
|
|
1141
1204
|
const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
|
|
1142
1205
|
const sentinelIndex = hotTokenIds.length;
|
|
1143
1206
|
const hotRowCount = sentinelIndex + 1;
|
|
@@ -1179,6 +1242,10 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1179
1242
|
}
|
|
1180
1243
|
|
|
1181
1244
|
const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
|
|
1245
|
+
const expandedF32Row = expandsF16ToF32
|
|
1246
|
+
? new Float32Array(totalPerLayerHiddenSize)
|
|
1247
|
+
: null;
|
|
1248
|
+
const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
|
|
1182
1249
|
for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
|
|
1183
1250
|
const tokenId = hotTokenIds[hotIndex];
|
|
1184
1251
|
const chunk = normalizeRangeBytes(
|
|
@@ -1197,6 +1264,21 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1197
1264
|
hiddenSizePerLayerInput * bytesPerElement
|
|
1198
1265
|
);
|
|
1199
1266
|
}
|
|
1267
|
+
} else if (sourceDtype === 'f16') {
|
|
1268
|
+
const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1269
|
+
for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
|
|
1270
|
+
expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
|
|
1271
|
+
}
|
|
1272
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
1273
|
+
const sourceStart = layerIdx * hiddenSizePerLayerInput;
|
|
1274
|
+
device.queue.writeBuffer(
|
|
1275
|
+
splitTables[layerIdx].buffer,
|
|
1276
|
+
hotIndex * hiddenSizePerLayerInput * bytesPerElement,
|
|
1277
|
+
expandedF32Row.buffer,
|
|
1278
|
+
sourceStart * bytesPerElement,
|
|
1279
|
+
hiddenSizePerLayerInput * bytesPerElement
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1200
1282
|
} else {
|
|
1201
1283
|
const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
|
|
1202
1284
|
for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
|
|
@@ -1221,7 +1303,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1221
1303
|
|
|
1222
1304
|
const runtime = {
|
|
1223
1305
|
mode: 'tokenizer_scores',
|
|
1224
|
-
maxTokens:
|
|
1306
|
+
maxTokens: capacity.maxHotTokens,
|
|
1307
|
+
maxBytes: policy.maxBytes,
|
|
1225
1308
|
outputDtype: policy.outputDtype,
|
|
1226
1309
|
vocabSize: vocabSizePerLayerInput,
|
|
1227
1310
|
numLayers,
|
|
@@ -1551,6 +1634,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1551
1634
|
hiddenSize: hiddenSizePerLayerInput,
|
|
1552
1635
|
vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
|
|
1553
1636
|
scaleEmbeddings: true,
|
|
1637
|
+
embeddingScale: null,
|
|
1554
1638
|
probeStage: 'per_layer_embed_out',
|
|
1555
1639
|
recorder,
|
|
1556
1640
|
numTokens,
|
|
@@ -164,8 +164,8 @@ async function readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer) {
|
|
|
164
164
|
return values;
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
-
function formatProbeStats(stats) {
|
|
168
|
-
|
|
167
|
+
function formatProbeStats(stats, firstNonFiniteIndex = -1) {
|
|
168
|
+
const fields = [
|
|
169
169
|
`min=${formatProbeNumber(stats.min)}`,
|
|
170
170
|
`max=${formatProbeNumber(stats.max)}`,
|
|
171
171
|
`mean=${formatProbeNumber(stats.mean)}`,
|
|
@@ -175,7 +175,11 @@ function formatProbeStats(stats) {
|
|
|
175
175
|
`nan=${stats.nanCount}`,
|
|
176
176
|
`inf=${stats.infCount}`,
|
|
177
177
|
`zero=${stats.zeroCount}`,
|
|
178
|
-
]
|
|
178
|
+
];
|
|
179
|
+
if (firstNonFiniteIndex >= 0) {
|
|
180
|
+
fields.push(`firstNonFinite=${firstNonFiniteIndex}`);
|
|
181
|
+
}
|
|
182
|
+
return fields.join(', ');
|
|
179
183
|
}
|
|
180
184
|
|
|
181
185
|
|
|
@@ -267,7 +271,8 @@ export async function runProbes(stage, buffer, options) {
|
|
|
267
271
|
let statsText = null;
|
|
268
272
|
if (includeStats) {
|
|
269
273
|
const row = await readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer);
|
|
270
|
-
|
|
274
|
+
const firstNonFiniteIndex = row.findIndex((value) => !Number.isFinite(value));
|
|
275
|
+
statsText = formatProbeStats(computeArrayStats(row), firstNonFiniteIndex);
|
|
271
276
|
}
|
|
272
277
|
|
|
273
278
|
const values = [];
|
|
@@ -28,14 +28,13 @@ import type { DopplerLoader } from '../../../loader/doppler-loader.js';
|
|
|
28
28
|
import type { LoadedEmbeddingPostprocessor } from '../../../loader/final-weights-loader.js';
|
|
29
29
|
import type { CompiledLayerPipeline } from './layer-plan.js';
|
|
30
30
|
import type { ExecutionPlanState } from './execution-plan.js';
|
|
31
|
-
import type {
|
|
31
|
+
import type { ExecutionV1CompiledState } from './execution-v1.js';
|
|
32
32
|
import type { LoRAAdapter } from './lora.js';
|
|
33
33
|
import type { RuntimeConfigSchema, KernelPathSchema } from '../../../config/schema/index.js';
|
|
34
34
|
import type { WeightDebugFlags } from './weights.js';
|
|
35
35
|
import type { LogitsDebugFlags } from './logits/index.js';
|
|
36
36
|
import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
|
|
37
|
-
import type { EmulationContext } from '
|
|
38
|
-
import type { PipelineStorageContext } from './init.js';
|
|
37
|
+
import type { EmulationContext, PipelineStorageContext } from './init.js';
|
|
39
38
|
import type { LinearAttentionRuntime } from './linear-attention.js';
|
|
40
39
|
import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
|
|
41
40
|
|
|
@@ -60,7 +59,7 @@ export class PipelineState {
|
|
|
60
59
|
resolvedKernelPath: KernelPathSchema | null;
|
|
61
60
|
kernelPathSource: KernelPathSource;
|
|
62
61
|
executionPlanState: ExecutionPlanState | null;
|
|
63
|
-
executionV1State:
|
|
62
|
+
executionV1State: ExecutionV1CompiledState | null;
|
|
64
63
|
disableRecordedLogits: boolean;
|
|
65
64
|
disableFusedDecode: boolean;
|
|
66
65
|
|
|
@@ -79,6 +78,7 @@ export class PipelineState {
|
|
|
79
78
|
|
|
80
79
|
// DopplerLoader instance
|
|
81
80
|
dopplerLoader: DopplerLoader | null;
|
|
81
|
+
ownsDopplerLoader: boolean;
|
|
82
82
|
|
|
83
83
|
// GPU context
|
|
84
84
|
gpuContext: { device?: GPUDevice } | null;
|
|
@@ -75,6 +75,7 @@ export class PipelineState {
|
|
|
75
75
|
// DopplerLoader instance
|
|
76
76
|
|
|
77
77
|
this.dopplerLoader = null;
|
|
78
|
+
this.ownsDopplerLoader = false;
|
|
78
79
|
|
|
79
80
|
// GPU context
|
|
80
81
|
|
|
@@ -111,6 +112,9 @@ export class PipelineState {
|
|
|
111
112
|
decodeReadbackCleanupMs: 0,
|
|
112
113
|
decodeReadbackCopyMs: 0,
|
|
113
114
|
prefillRecordMs: 0,
|
|
115
|
+
prefillRecordOps: 0,
|
|
116
|
+
prefillRecordPasses: 0,
|
|
117
|
+
prefillRecordOpLabels: {},
|
|
114
118
|
prefillSubmitWaitMs: 0,
|
|
115
119
|
prefillProfileSteps: [],
|
|
116
120
|
decodeProfileSteps: [],
|
|
@@ -46,6 +46,10 @@ export interface KVCacheSnapshot {
|
|
|
46
46
|
linearAttention?: LinearAttentionRuntime | null;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
export interface WorkloadPhaseTiming {
|
|
50
|
+
[key: string]: number | string | boolean | null | Record<string, number> | WorkloadPhaseTiming | WorkloadPhaseTiming[];
|
|
51
|
+
}
|
|
52
|
+
|
|
49
53
|
export interface AdvanceEmbeddingResult {
|
|
50
54
|
embedding: Float32Array;
|
|
51
55
|
embeddingMode: 'last' | 'mean';
|
|
@@ -175,6 +179,12 @@ export interface LayerContext {
|
|
|
175
179
|
};
|
|
176
180
|
captureConfig?: Record<string, unknown>;
|
|
177
181
|
} | null;
|
|
182
|
+
/** Opt into fused gate/up GeGLU prefill path when runtime profile enables it. */
|
|
183
|
+
useFusedGateUpGelu?: boolean;
|
|
184
|
+
/** Opt into large-batch f16-weight/f32-activation fused gate/up prefill. */
|
|
185
|
+
useLargeBatchF16F32FusedGateUp?: boolean;
|
|
186
|
+
/** Skip KV cache writes for hidden-state-only prefill routes. */
|
|
187
|
+
skipKVCacheWrites?: boolean;
|
|
178
188
|
}
|
|
179
189
|
|
|
180
190
|
/**
|
|
@@ -408,6 +418,8 @@ export interface LogitsStepResult {
|
|
|
408
418
|
export interface PrefillResult extends KVCacheSnapshot {
|
|
409
419
|
/** Finalized logits for the next token after prefill */
|
|
410
420
|
logits: Float32Array;
|
|
421
|
+
|
|
422
|
+
phase?: WorkloadPhaseTiming | null;
|
|
411
423
|
}
|
|
412
424
|
|
|
413
425
|
/**
|
|
@@ -425,6 +437,8 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
|
|
|
425
437
|
|
|
426
438
|
/** Pooling mode used to construct embedding */
|
|
427
439
|
embeddingMode: 'last' | 'mean';
|
|
440
|
+
|
|
441
|
+
phase?: WorkloadPhaseTiming | null;
|
|
428
442
|
}
|
|
429
443
|
|
|
430
444
|
/**
|
|
@@ -485,6 +499,7 @@ export interface LayerWeights {
|
|
|
485
499
|
kProj: LayerWeightBuffer;
|
|
486
500
|
vProj?: LayerWeightBuffer;
|
|
487
501
|
oProj: LayerWeightBuffer;
|
|
502
|
+
qGateProj?: LayerWeightBuffer | null;
|
|
488
503
|
convInProj?: LayerWeightBuffer;
|
|
489
504
|
convKernel?: LayerWeightBuffer;
|
|
490
505
|
convOutProj?: LayerWeightBuffer;
|
|
@@ -492,8 +507,12 @@ export interface LayerWeights {
|
|
|
492
507
|
qkvProj?: GPUBuffer | WeightBuffer | null;
|
|
493
508
|
/** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
|
|
494
509
|
qkvSizes?: [number, number, number];
|
|
495
|
-
/** Data type of fused QKV weights (f16 or
|
|
496
|
-
qkvDtype?: 'f16' | 'f32';
|
|
510
|
+
/** Data type of fused QKV weights (f16, f32, or q4k) */
|
|
511
|
+
qkvDtype?: 'f16' | 'f32' | 'q4k';
|
|
512
|
+
/** Fused linear-attention A/B projection weight for decode-only A+B projection fusion. */
|
|
513
|
+
linearABProj?: WeightBuffer | null;
|
|
514
|
+
/** Fused linear-attention QKV/Z projection weight for decode-only QKV+Z projection fusion. */
|
|
515
|
+
linearQKVZProj?: WeightBuffer | null;
|
|
497
516
|
|
|
498
517
|
// FFN (dense layers)
|
|
499
518
|
postAttentionNorm?: GPUBuffer | Float32Array;
|
|
@@ -610,6 +629,10 @@ export interface PipelineStats {
|
|
|
610
629
|
decodeRecordOps?: number;
|
|
611
630
|
decodeRecordPasses?: number;
|
|
612
631
|
decodeRecordOpLabels?: Record<string, number>;
|
|
632
|
+
prefillRecordMs?: number;
|
|
633
|
+
prefillRecordOps?: number;
|
|
634
|
+
prefillRecordPasses?: number;
|
|
635
|
+
prefillRecordOpLabels?: Record<string, number>;
|
|
613
636
|
uniformCache?: UniformCacheStats | null;
|
|
614
637
|
decodeSubmitWaitMs?: number;
|
|
615
638
|
decodeReadbackWaitMs?: number;
|
|
@@ -640,6 +663,7 @@ export interface PipelineStats {
|
|
|
640
663
|
activationDtype: string;
|
|
641
664
|
readbackInterval: number | null;
|
|
642
665
|
readbackMode: string | null;
|
|
666
|
+
maxBatchDecodeTokens?: number | null;
|
|
643
667
|
batchSize: number;
|
|
644
668
|
stopCheckMode: string;
|
|
645
669
|
disableCommandBatching?: boolean;
|
|
@@ -654,6 +678,7 @@ export interface PipelineStats {
|
|
|
654
678
|
activationDtype: string;
|
|
655
679
|
readbackInterval: number | null;
|
|
656
680
|
readbackMode: string | null;
|
|
681
|
+
maxBatchDecodeTokens?: number | null;
|
|
657
682
|
batchSize: number;
|
|
658
683
|
stopCheckMode: string;
|
|
659
684
|
disableCommandBatching?: boolean;
|
|
@@ -42,14 +42,19 @@ export function getWeightBuffer(weight, label, deviceOverride = null) {
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
let data;
|
|
45
|
-
let bufferDtype
|
|
45
|
+
let bufferDtype;
|
|
46
46
|
if (isCpuWeightBuffer(weight)) {
|
|
47
47
|
data = weight.data;
|
|
48
|
-
|
|
48
|
+
if (!weight.dtype) {
|
|
49
|
+
throw new Error(`Weight buffer "${label}" is missing dtype metadata.`);
|
|
50
|
+
}
|
|
51
|
+
bufferDtype = weight.dtype;
|
|
49
52
|
} else if (weight instanceof Float32Array) {
|
|
50
53
|
data = weight;
|
|
54
|
+
bufferDtype = 'f32';
|
|
51
55
|
} else {
|
|
52
56
|
data = new Float32Array( (weight));
|
|
57
|
+
bufferDtype = 'f32';
|
|
53
58
|
}
|
|
54
59
|
|
|
55
60
|
const buf = acquireBuffer(data.byteLength, undefined, label);
|
|
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
|
|
|
15
15
|
import { PipelineGenerator } from './text/generator.js';
|
|
16
16
|
import type { Manifest } from './text/config.js';
|
|
17
17
|
import type { WeightLoadResult, PipelineContexts } from './text/init.js';
|
|
18
|
-
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats } from './text/types.js';
|
|
18
|
+
import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
|
|
19
19
|
import type { ChatMessage } from './text/chat-format.js';
|
|
20
20
|
import type { LoRAAdapter } from './text/lora.js';
|
|
21
21
|
import type { DiffusionPipeline } from './diffusion/pipeline.js';
|
|
@@ -120,6 +120,7 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
120
120
|
tokens: number[];
|
|
121
121
|
seqLen: number;
|
|
122
122
|
embeddingMode: string;
|
|
123
|
+
phase?: WorkloadPhaseTiming | null;
|
|
123
124
|
}>;
|
|
124
125
|
|
|
125
126
|
embedBatch(prompts: string[], options?: GenerateOptions): Promise<Array<{
|
|
@@ -131,6 +132,25 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
131
132
|
|
|
132
133
|
prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
|
|
133
134
|
|
|
135
|
+
prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
136
|
+
seqLen: number;
|
|
137
|
+
tokens: number[];
|
|
138
|
+
tokenIds: number[];
|
|
139
|
+
logits: Float32Array;
|
|
140
|
+
logitsByTokenId: Record<number, number>;
|
|
141
|
+
phase?: WorkloadPhaseTiming | null;
|
|
142
|
+
}>;
|
|
143
|
+
|
|
144
|
+
prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
|
|
145
|
+
seqLen: number;
|
|
146
|
+
prefixTokens: number[];
|
|
147
|
+
tokens: number[];
|
|
148
|
+
tokenIds: number[];
|
|
149
|
+
logits: Float32Array;
|
|
150
|
+
logitsByTokenId: Record<number, number>;
|
|
151
|
+
phase?: WorkloadPhaseTiming | null;
|
|
152
|
+
}>;
|
|
153
|
+
|
|
134
154
|
applyKVCacheSnapshot(snapshot: KVCacheSnapshot): void;
|
|
135
155
|
|
|
136
156
|
generateWithPrefixKV(
|