doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
import { getBufferDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
recordMatmul,
|
|
4
|
+
recordRMSNorm,
|
|
5
|
+
runMatmul,
|
|
6
|
+
runRMSNorm,
|
|
7
|
+
castF32ToF16,
|
|
8
|
+
castF16ToF32,
|
|
9
|
+
recordCastF32ToF16,
|
|
10
|
+
recordCastF16ToF32,
|
|
11
|
+
} from '../../../gpu/kernel-selector.js';
|
|
3
12
|
import { readBuffer, releaseBuffer, uploadData, acquireBuffer } from '../../../memory/buffer-pool.js';
|
|
4
13
|
import { log } from '../../../debug/index.js';
|
|
5
14
|
import { decodeReadback, f16ToF32 } from './debug-utils/index.js';
|
|
6
15
|
import { runLinearAttentionCoreGPU } from '../../../gpu/kernels/linear-attention-core.js';
|
|
16
|
+
import {
|
|
17
|
+
resolveLinearAttentionABProjection,
|
|
18
|
+
resolveLinearAttentionQKVZProjection,
|
|
19
|
+
} from './linear-attention-ab-fusion.js';
|
|
7
20
|
import { runProbes } from './probes.js';
|
|
8
21
|
import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
|
|
9
22
|
import { dequantizeQ4KM } from '../../../converter/quantizer.js';
|
|
@@ -34,9 +47,7 @@ function normalizeLinearNormMode(value) {
|
|
|
34
47
|
}
|
|
35
48
|
|
|
36
49
|
function bytesFromDtype(dtype) {
|
|
37
|
-
|
|
38
|
-
if (normalized === 'f16' || normalized === 'bf16') return 2;
|
|
39
|
-
return 4;
|
|
50
|
+
return selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
40
51
|
}
|
|
41
52
|
|
|
42
53
|
function resolveMatmulStepDtype(role, phase, layerIdx, kernelPath, fallback, field) {
|
|
@@ -199,13 +210,19 @@ function inferLinearNormModeFromWeight(weight, projectionLayout) {
|
|
|
199
210
|
if (weight instanceof ArrayBuffer) {
|
|
200
211
|
return classify(Math.trunc(weight.byteLength / Float32Array.BYTES_PER_ELEMENT));
|
|
201
212
|
}
|
|
213
|
+
if (!weight || typeof weight !== 'object') {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
202
216
|
const explicitDtype = typeof weight?.dtype === 'string' ? weight.dtype.toLowerCase() : null;
|
|
203
217
|
const trackedDtype = isGpuBuffer(weight) ? String(getBufferDtype(weight) ?? '').toLowerCase() : '';
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
218
|
+
const sourceDtype = explicitDtype || trackedDtype;
|
|
219
|
+
if (!sourceDtype || !Number.isFinite(weight?.size)) {
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
const bytesPerElement = bytesFromDtype(sourceDtype);
|
|
223
|
+
const size = Number(weight.size);
|
|
224
|
+
const sizedElements = Math.trunc(size / bytesPerElement);
|
|
225
|
+
if (sizedElements && size % bytesPerElement === 0) {
|
|
209
226
|
return classify(sizedElements);
|
|
210
227
|
}
|
|
211
228
|
return null;
|
|
@@ -652,9 +669,32 @@ async function projectLinearTensor({
|
|
|
652
669
|
? await recordCastF32ToF16(recorder, inputTensor)
|
|
653
670
|
: await castF32ToF16(inputTensor);
|
|
654
671
|
}
|
|
672
|
+
const settleProjectionOutputDtype = async (result) => {
|
|
673
|
+
if (result.dtype === resolvedOutputDtype) {
|
|
674
|
+
return result;
|
|
675
|
+
}
|
|
676
|
+
assertImplicitDtypeTransitionAllowed({
|
|
677
|
+
executionPolicies,
|
|
678
|
+
fromDtype: result.dtype,
|
|
679
|
+
toDtype: resolvedOutputDtype,
|
|
680
|
+
op: role,
|
|
681
|
+
detail: 'Linear attention projection returned a kernel-selected dtype different from the declared projection output dtype.',
|
|
682
|
+
});
|
|
683
|
+
if (resolvedOutputDtype === 'f16') {
|
|
684
|
+
const casted = recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result);
|
|
685
|
+
releaseOrTrackBuffer(recorder, result.buffer);
|
|
686
|
+
return casted;
|
|
687
|
+
}
|
|
688
|
+
if (resolvedOutputDtype === 'f32') {
|
|
689
|
+
const casted = recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result);
|
|
690
|
+
releaseOrTrackBuffer(recorder, result.buffer);
|
|
691
|
+
return casted;
|
|
692
|
+
}
|
|
693
|
+
throw new Error(`Unsupported linear_attention projection output dtype "${String(resolvedOutputDtype)}".`);
|
|
694
|
+
};
|
|
655
695
|
try {
|
|
656
696
|
if (recorder) {
|
|
657
|
-
|
|
697
|
+
const result = await recordMatmul(recorder, matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
|
|
658
698
|
transposeB: 'auto',
|
|
659
699
|
role,
|
|
660
700
|
layerIdx,
|
|
@@ -662,8 +702,9 @@ async function projectLinearTensor({
|
|
|
662
702
|
outputDtype: resolvedOutputDtype,
|
|
663
703
|
executionPolicies,
|
|
664
704
|
});
|
|
705
|
+
return await settleProjectionOutputDtype(result);
|
|
665
706
|
}
|
|
666
|
-
|
|
707
|
+
const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
|
|
667
708
|
transposeB: 'auto',
|
|
668
709
|
role,
|
|
669
710
|
layerIdx,
|
|
@@ -671,6 +712,7 @@ async function projectLinearTensor({
|
|
|
671
712
|
outputDtype: resolvedOutputDtype,
|
|
672
713
|
executionPolicies,
|
|
673
714
|
});
|
|
715
|
+
return await settleProjectionOutputDtype(result);
|
|
674
716
|
} finally {
|
|
675
717
|
if (matmulInput !== inputTensor) {
|
|
676
718
|
releaseOrTrackBuffer(recorder, matmulInput.buffer);
|
|
@@ -679,6 +721,31 @@ async function projectLinearTensor({
|
|
|
679
721
|
}
|
|
680
722
|
}
|
|
681
723
|
|
|
724
|
+
async function settleLinearAttentionCoreInputDtype(tensor, targetDtype, options) {
|
|
725
|
+
if (tensor.dtype === targetDtype) {
|
|
726
|
+
return tensor;
|
|
727
|
+
}
|
|
728
|
+
const {
|
|
729
|
+
recorder,
|
|
730
|
+
executionPolicies,
|
|
731
|
+
role,
|
|
732
|
+
} = options;
|
|
733
|
+
assertImplicitDtypeTransitionAllowed({
|
|
734
|
+
executionPolicies,
|
|
735
|
+
fromDtype: tensor.dtype,
|
|
736
|
+
toDtype: targetDtype,
|
|
737
|
+
op: role,
|
|
738
|
+
detail: 'Linear attention core requires qkv/z/a/b tensors to share one input dtype.',
|
|
739
|
+
});
|
|
740
|
+
if (targetDtype === 'f16') {
|
|
741
|
+
return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
|
|
742
|
+
}
|
|
743
|
+
if (targetDtype === 'f32') {
|
|
744
|
+
return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
|
|
745
|
+
}
|
|
746
|
+
throw new Error(`Unsupported linear_attention core input dtype "${String(targetDtype)}".`);
|
|
747
|
+
}
|
|
748
|
+
|
|
682
749
|
export function hasLinearAttentionLayers(layerTypes) {
|
|
683
750
|
if (!Array.isArray(layerTypes)) return false;
|
|
684
751
|
for (let i = 0; i < layerTypes.length; i++) {
|
|
@@ -767,6 +834,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
767
834
|
getNormWeightBuffer,
|
|
768
835
|
recorder,
|
|
769
836
|
executionPolicies = null,
|
|
837
|
+
precomputedInputNorm = null,
|
|
770
838
|
} = options;
|
|
771
839
|
|
|
772
840
|
if (!layerWeights) {
|
|
@@ -804,7 +872,20 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
804
872
|
let normedTensor = inputTensor;
|
|
805
873
|
let normedCreated = false;
|
|
806
874
|
|
|
807
|
-
if (
|
|
875
|
+
if (precomputedInputNorm) {
|
|
876
|
+
if (
|
|
877
|
+
phase !== 'decode'
|
|
878
|
+
|| numTokens !== 1
|
|
879
|
+
|| !layerWeights.inputNorm
|
|
880
|
+
|| precomputedInputNorm.dtype !== inputTensor.dtype
|
|
881
|
+
|| precomputedInputNorm.dtype !== 'f32'
|
|
882
|
+
) {
|
|
883
|
+
releaseOrTrackBuffer(recorder, precomputedInputNorm.buffer);
|
|
884
|
+
throw new Error(`linear_attention layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
885
|
+
}
|
|
886
|
+
normedTensor = precomputedInputNorm;
|
|
887
|
+
normedCreated = true;
|
|
888
|
+
} else if (layerWeights.inputNorm) {
|
|
808
889
|
const normWeightBuffer = getNormWeightBuffer(layerWeights.inputNorm, `L${layerIdx}.linear_input_norm`);
|
|
809
890
|
try {
|
|
810
891
|
if (recorder) {
|
|
@@ -828,7 +909,35 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
828
909
|
}
|
|
829
910
|
}
|
|
830
911
|
|
|
831
|
-
const
|
|
912
|
+
const qkvzProjection = resolveLinearAttentionQKVZProjection(layerWeights, {
|
|
913
|
+
phase,
|
|
914
|
+
numTokens,
|
|
915
|
+
hiddenSize,
|
|
916
|
+
numVHeads: projectionLayout.numVHeads,
|
|
917
|
+
convDim: projectionLayout.convDim,
|
|
918
|
+
valueDim: projectionLayout.valueDim,
|
|
919
|
+
layerIdx,
|
|
920
|
+
debugProbes: options.debugProbes,
|
|
921
|
+
operatorDiagnostics: options.operatorDiagnostics,
|
|
922
|
+
});
|
|
923
|
+
const qkvzTensor = qkvzProjection
|
|
924
|
+
? await projectLinearTensor({
|
|
925
|
+
inputTensor: normedTensor,
|
|
926
|
+
sourceWeight: qkvzProjection.weight,
|
|
927
|
+
role: 'linear_qkvz_proj',
|
|
928
|
+
phase,
|
|
929
|
+
outDim: qkvzProjection.outDim,
|
|
930
|
+
numTokens,
|
|
931
|
+
hiddenSize,
|
|
932
|
+
layerIdx,
|
|
933
|
+
kernelPath,
|
|
934
|
+
outputDtype: projectionDtype,
|
|
935
|
+
getWeightBuffer,
|
|
936
|
+
recorder,
|
|
937
|
+
executionPolicies,
|
|
938
|
+
})
|
|
939
|
+
: null;
|
|
940
|
+
const qkvTensor = qkvzTensor ?? await projectLinearTensor({
|
|
832
941
|
inputTensor: normedTensor,
|
|
833
942
|
sourceWeight: layerWeights.qkvProj,
|
|
834
943
|
role: 'linear_qkv_proj',
|
|
@@ -843,7 +952,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
843
952
|
recorder,
|
|
844
953
|
executionPolicies,
|
|
845
954
|
});
|
|
846
|
-
const zTensor = await projectLinearTensor({
|
|
955
|
+
const zTensor = qkvzTensor ?? await projectLinearTensor({
|
|
847
956
|
inputTensor: normedTensor,
|
|
848
957
|
sourceWeight: layerWeights.linearInProjZ,
|
|
849
958
|
role: 'linear_z_proj',
|
|
@@ -858,7 +967,33 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
858
967
|
recorder,
|
|
859
968
|
executionPolicies,
|
|
860
969
|
});
|
|
861
|
-
const
|
|
970
|
+
const abProjection = resolveLinearAttentionABProjection(layerWeights, {
|
|
971
|
+
phase,
|
|
972
|
+
numTokens,
|
|
973
|
+
hiddenSize,
|
|
974
|
+
numVHeads: projectionLayout.numVHeads,
|
|
975
|
+
layerIdx,
|
|
976
|
+
debugProbes: options.debugProbes,
|
|
977
|
+
operatorDiagnostics: options.operatorDiagnostics,
|
|
978
|
+
});
|
|
979
|
+
const abTensor = abProjection
|
|
980
|
+
? await projectLinearTensor({
|
|
981
|
+
inputTensor: normedTensor,
|
|
982
|
+
sourceWeight: abProjection.weight,
|
|
983
|
+
role: 'linear_ab_proj',
|
|
984
|
+
phase,
|
|
985
|
+
outDim: abProjection.outDim,
|
|
986
|
+
numTokens,
|
|
987
|
+
hiddenSize,
|
|
988
|
+
layerIdx,
|
|
989
|
+
kernelPath,
|
|
990
|
+
outputDtype: projectionDtype,
|
|
991
|
+
getWeightBuffer,
|
|
992
|
+
recorder,
|
|
993
|
+
executionPolicies,
|
|
994
|
+
})
|
|
995
|
+
: null;
|
|
996
|
+
const aTensor = abTensor ?? await projectLinearTensor({
|
|
862
997
|
inputTensor: normedTensor,
|
|
863
998
|
sourceWeight: layerWeights.linearInProjA,
|
|
864
999
|
role: 'linear_a_proj',
|
|
@@ -873,7 +1008,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
873
1008
|
recorder,
|
|
874
1009
|
executionPolicies,
|
|
875
1010
|
});
|
|
876
|
-
const bTensor = await projectLinearTensor({
|
|
1011
|
+
const bTensor = abTensor ?? await projectLinearTensor({
|
|
877
1012
|
inputTensor: normedTensor,
|
|
878
1013
|
sourceWeight: layerWeights.linearInProjB,
|
|
879
1014
|
role: 'linear_b_proj',
|
|
@@ -905,6 +1040,10 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
905
1040
|
layerOutputDtype,
|
|
906
1041
|
'outputDtype'
|
|
907
1042
|
);
|
|
1043
|
+
let coreQkvTensor = qkvTensor;
|
|
1044
|
+
let coreZTensor = zTensor;
|
|
1045
|
+
let coreATensor = aTensor;
|
|
1046
|
+
let coreBTensor = bTensor;
|
|
908
1047
|
|
|
909
1048
|
try {
|
|
910
1049
|
await runProbes('linear_qkv_proj', qkvTensor.buffer, {
|
|
@@ -943,11 +1082,27 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
943
1082
|
operatorDiagnostics: options.operatorDiagnostics,
|
|
944
1083
|
dtype: bTensor.dtype,
|
|
945
1084
|
});
|
|
1085
|
+
const coreInputDtype = qkvTensor.dtype;
|
|
1086
|
+
coreZTensor = await settleLinearAttentionCoreInputDtype(zTensor, coreInputDtype, {
|
|
1087
|
+
recorder,
|
|
1088
|
+
executionPolicies,
|
|
1089
|
+
role: 'linear_z_proj',
|
|
1090
|
+
});
|
|
1091
|
+
coreATensor = await settleLinearAttentionCoreInputDtype(aTensor, coreInputDtype, {
|
|
1092
|
+
recorder,
|
|
1093
|
+
executionPolicies,
|
|
1094
|
+
role: 'linear_a_proj',
|
|
1095
|
+
});
|
|
1096
|
+
coreBTensor = await settleLinearAttentionCoreInputDtype(bTensor, coreInputDtype, {
|
|
1097
|
+
recorder,
|
|
1098
|
+
executionPolicies,
|
|
1099
|
+
role: 'linear_b_proj',
|
|
1100
|
+
});
|
|
946
1101
|
const coreTensor = await runLinearAttentionCoreGPU(
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
1102
|
+
coreQkvTensor,
|
|
1103
|
+
coreZTensor,
|
|
1104
|
+
coreATensor,
|
|
1105
|
+
coreBTensor,
|
|
951
1106
|
layerState,
|
|
952
1107
|
{
|
|
953
1108
|
numTokens,
|
|
@@ -956,6 +1111,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
956
1111
|
qkL2NormEps: QK_L2NORM_EPS,
|
|
957
1112
|
recorder,
|
|
958
1113
|
executionPolicies,
|
|
1114
|
+
abPacked: abProjection != null,
|
|
1115
|
+
qkvzPacked: qkvzProjection != null,
|
|
1116
|
+
bProjOffsetElements: abProjection?.bProjOffsetElements ?? 0,
|
|
959
1117
|
}
|
|
960
1118
|
);
|
|
961
1119
|
await runProbes('linear_core_out', coreTensor.buffer, {
|
|
@@ -1013,9 +1171,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
1013
1171
|
if (normedCreated) {
|
|
1014
1172
|
releaseOrTrackBuffer(recorder, normedTensor.buffer);
|
|
1015
1173
|
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1174
|
+
const released = new Set();
|
|
1175
|
+
for (const tensor of [qkvTensor, zTensor, aTensor, bTensor, coreQkvTensor, coreZTensor, coreATensor, coreBTensor]) {
|
|
1176
|
+
const buffer = tensor?.buffer ?? null;
|
|
1177
|
+
if (!buffer || released.has(buffer)) continue;
|
|
1178
|
+
released.add(buffer);
|
|
1179
|
+
releaseOrTrackBuffer(recorder, buffer);
|
|
1180
|
+
}
|
|
1020
1181
|
}
|
|
1021
1182
|
}
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
|
|
4
4
|
import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
|
|
5
|
-
import { runMatmul, runRMSNorm } from '../../../../gpu/kernel-selector.js';
|
|
5
|
+
import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
|
|
6
6
|
import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
|
|
7
7
|
import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
|
|
8
|
-
import {
|
|
8
|
+
import { recordLmHeadArgmax } from '../../../../gpu/kernels/lm-head-argmax.js';
|
|
9
9
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
10
10
|
import {
|
|
11
11
|
castF16ToF32,
|
|
@@ -21,25 +21,9 @@ import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
|
21
21
|
import { runProbes } from '../probes.js';
|
|
22
22
|
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
23
23
|
import { f16BufferToF32 } from './cpu.js';
|
|
24
|
-
import { readBufferWithCleanup } from './utils.js';
|
|
24
|
+
import { readBufferWithCleanup, resolveLogitInputScale } from './utils.js';
|
|
25
25
|
import { f16ToF32 } from '../../../../loader/dtype-utils.js';
|
|
26
|
-
|
|
27
|
-
function shouldForceStableF32Logits(config, inputDtype) {
|
|
28
|
-
if (inputDtype !== 'f16') {
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
// Softcapped output heads are numerically sensitive in pure F16 on the
|
|
32
|
-
// final RMSNorm + LM-head path. Widen only the logits tail so the main
|
|
33
|
-
// layer stack and KV cache can stay on the faster F16 lane.
|
|
34
|
-
if (Number.isFinite(config.finalLogitSoftcapping) && config.finalLogitSoftcapping > 0) {
|
|
35
|
-
return true;
|
|
36
|
-
}
|
|
37
|
-
// Small Gemma-family checkpoints can also overflow in pure F16 logits path
|
|
38
|
-
// after RMSNorm offset even without output softcapping.
|
|
39
|
-
return config.rmsNormWeightOffset === true
|
|
40
|
-
&& Number.isFinite(config.hiddenSize)
|
|
41
|
-
&& config.hiddenSize <= 768;
|
|
42
|
-
}
|
|
26
|
+
import { shouldForceStableF32Logits, createStableF32LogitsKernelPath } from './precision-policy.js';
|
|
43
27
|
|
|
44
28
|
function resolvePrecisionFieldDtype(precision, fallback, field) {
|
|
45
29
|
const requested = precision?.[field] ?? fallback;
|
|
@@ -84,68 +68,6 @@ async function coerceTensorDtype(tensor, targetDtype, recorder = null, options =
|
|
|
84
68
|
throw new Error(`Unsupported logits matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
|
|
85
69
|
}
|
|
86
70
|
|
|
87
|
-
const STABLE_F32_LOGITS_KERNEL_MAP = new Map([
|
|
88
|
-
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
89
|
-
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
90
|
-
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
91
|
-
]);
|
|
92
|
-
|
|
93
|
-
function createStableF32LogitsKernelPath(kernelPath) {
|
|
94
|
-
if (!kernelPath?.postLayer) {
|
|
95
|
-
return kernelPath;
|
|
96
|
-
}
|
|
97
|
-
let changed = false;
|
|
98
|
-
const postLayer = kernelPath.postLayer.map((step) => {
|
|
99
|
-
if (step?.op === 'final_norm') {
|
|
100
|
-
const precision = {
|
|
101
|
-
...(step.precision ?? {}),
|
|
102
|
-
inputDtype: 'f32',
|
|
103
|
-
outputDtype: 'f32',
|
|
104
|
-
};
|
|
105
|
-
if (
|
|
106
|
-
step.precision?.inputDtype === precision.inputDtype
|
|
107
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
108
|
-
) {
|
|
109
|
-
return step;
|
|
110
|
-
}
|
|
111
|
-
changed = true;
|
|
112
|
-
return {
|
|
113
|
-
...step,
|
|
114
|
-
precision,
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
if (step?.op !== 'lm_head' && step?.op !== 'lm_head_prefill') {
|
|
118
|
-
return step;
|
|
119
|
-
}
|
|
120
|
-
const replacement = STABLE_F32_LOGITS_KERNEL_MAP.get(step.kernel) ?? step.kernel;
|
|
121
|
-
const precision = {
|
|
122
|
-
...(step.precision ?? {}),
|
|
123
|
-
inputDtype: 'f32',
|
|
124
|
-
outputDtype: 'f32',
|
|
125
|
-
};
|
|
126
|
-
if (
|
|
127
|
-
replacement === step.kernel
|
|
128
|
-
&& step.precision?.inputDtype === precision.inputDtype
|
|
129
|
-
&& step.precision?.outputDtype === precision.outputDtype
|
|
130
|
-
) {
|
|
131
|
-
return step;
|
|
132
|
-
}
|
|
133
|
-
changed = true;
|
|
134
|
-
return {
|
|
135
|
-
...step,
|
|
136
|
-
kernel: replacement,
|
|
137
|
-
precision,
|
|
138
|
-
};
|
|
139
|
-
});
|
|
140
|
-
if (!changed) {
|
|
141
|
-
return kernelPath;
|
|
142
|
-
}
|
|
143
|
-
return {
|
|
144
|
-
...kernelPath,
|
|
145
|
-
postLayer,
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
|
|
149
71
|
const bf16ScratchU32 = new Uint32Array(1);
|
|
150
72
|
const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
|
|
151
73
|
const SPLIT_UPLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
@@ -175,6 +97,26 @@ function writeBufferInChunks(queue, buffer, bytes) {
|
|
|
175
97
|
}
|
|
176
98
|
}
|
|
177
99
|
|
|
100
|
+
function resolveFinalNormGpuBuffer(finalNorm, queue, label) {
|
|
101
|
+
if (isWeightBuffer(finalNorm)) {
|
|
102
|
+
return { buffer: finalNorm.buffer, owned: false };
|
|
103
|
+
}
|
|
104
|
+
if (isGpuBufferInstance(finalNorm)) {
|
|
105
|
+
return { buffer: finalNorm, owned: false };
|
|
106
|
+
}
|
|
107
|
+
if (!ArrayBuffer.isView(finalNorm)) {
|
|
108
|
+
throw new Error('[Logits] final_norm must be a GPU buffer, typed array, or WeightBuffer.');
|
|
109
|
+
}
|
|
110
|
+
const buffer = acquireBuffer(finalNorm.byteLength, undefined, label);
|
|
111
|
+
try {
|
|
112
|
+
queue.writeBuffer(buffer, 0, finalNorm);
|
|
113
|
+
return { buffer, owned: true };
|
|
114
|
+
} catch (error) {
|
|
115
|
+
releaseBuffer(buffer);
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
178
120
|
function normalizeRangeBytes(value, label) {
|
|
179
121
|
if (value instanceof Uint8Array) return value;
|
|
180
122
|
if (value instanceof ArrayBuffer) return new Uint8Array(value);
|
|
@@ -304,9 +246,12 @@ export async function extractLmHeadChunk(
|
|
|
304
246
|
vocabSize,
|
|
305
247
|
rowOffset,
|
|
306
248
|
rowCount,
|
|
307
|
-
sourceDtype
|
|
249
|
+
sourceDtype
|
|
308
250
|
) {
|
|
309
|
-
|
|
251
|
+
if (typeof sourceDtype !== 'string' || sourceDtype.trim() === '') {
|
|
252
|
+
throw new Error('[Logits] CPU LM head source dtype is required.');
|
|
253
|
+
}
|
|
254
|
+
const normalizedSourceDtype = sourceDtype.trim().toLowerCase();
|
|
310
255
|
if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
|
|
311
256
|
throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
|
|
312
257
|
}
|
|
@@ -800,6 +745,8 @@ export async function computeLogitsGPU(
|
|
|
800
745
|
let normInputOwned = false;
|
|
801
746
|
let normedTensor;
|
|
802
747
|
let finalNormTensor;
|
|
748
|
+
let logitInputTensor;
|
|
749
|
+
let logitInputOwned = false;
|
|
803
750
|
let lmHeadInputTensor;
|
|
804
751
|
let lmHeadInputOwned = false;
|
|
805
752
|
let lmHeadBuffer;
|
|
@@ -815,13 +762,9 @@ export async function computeLogitsGPU(
|
|
|
815
762
|
}
|
|
816
763
|
|
|
817
764
|
// Apply final RMSNorm
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
|
|
822
|
-
device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
|
|
823
|
-
normWeightBufferOwned = true;
|
|
824
|
-
}
|
|
765
|
+
const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, device.queue, 'final_norm_w');
|
|
766
|
+
normWeightBuffer = resolvedFinalNorm.buffer;
|
|
767
|
+
normWeightBufferOwned = resolvedFinalNorm.owned;
|
|
825
768
|
|
|
826
769
|
const inputDtype = isGpuBufferInstance(hiddenStates) ? activationDtype : 'f32';
|
|
827
770
|
const inputTensor = createTensor(inputBuffer, inputDtype, [numTokens, hiddenSize], 'logits_input');
|
|
@@ -899,21 +842,29 @@ export async function computeLogitsGPU(
|
|
|
899
842
|
releaseBuffer(normInputTensor.buffer);
|
|
900
843
|
normInputOwned = false;
|
|
901
844
|
}
|
|
845
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
846
|
+
logitInputTensor = finalNormTensor;
|
|
847
|
+
if (logitInputScale !== 1) {
|
|
848
|
+
logitInputTensor = await runScale(finalNormTensor, logitInputScale, {
|
|
849
|
+
count: numTokens * hiddenSize,
|
|
850
|
+
});
|
|
851
|
+
logitInputOwned = true;
|
|
852
|
+
}
|
|
902
853
|
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
903
854
|
const lmHeadInputDtype = forceStableF32Logits
|
|
904
|
-
?
|
|
905
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
855
|
+
? logitInputTensor.dtype
|
|
856
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
|
|
906
857
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
907
|
-
?
|
|
908
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
909
|
-
lmHeadInputTensor = lmHeadInputDtype !==
|
|
910
|
-
? await coerceTensorDtype(
|
|
858
|
+
? logitInputTensor.dtype
|
|
859
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
|
|
860
|
+
lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
|
|
861
|
+
? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, null, {
|
|
911
862
|
executionPolicies: config.executionPolicies ?? null,
|
|
912
863
|
op: 'lm_head',
|
|
913
864
|
transitionDeclaredBy: 'step_precision',
|
|
914
865
|
})
|
|
915
|
-
:
|
|
916
|
-
lmHeadInputOwned = lmHeadInputTensor !==
|
|
866
|
+
: logitInputTensor;
|
|
867
|
+
lmHeadInputOwned = lmHeadInputTensor !== logitInputTensor;
|
|
917
868
|
|
|
918
869
|
// Project to vocab via LM head
|
|
919
870
|
if (isGpuBufferInstance(lmHead)) {
|
|
@@ -948,6 +899,7 @@ export async function computeLogitsGPU(
|
|
|
948
899
|
// Cleanup intermediate buffers (but keep logitsBuffer)
|
|
949
900
|
if (inputBufferOwned) { releaseBuffer(inputBuffer); inputBufferOwned = false; }
|
|
950
901
|
if (lmHeadInputOwned) { releaseBuffer(lmHeadInputTensor.buffer); lmHeadInputOwned = false; }
|
|
902
|
+
if (logitInputOwned) { releaseBuffer(logitInputTensor.buffer); logitInputOwned = false; }
|
|
951
903
|
if (finalNormTensor) {
|
|
952
904
|
releaseBuffer(finalNormTensor.buffer);
|
|
953
905
|
finalNormTensor = null;
|
|
@@ -961,6 +913,7 @@ export async function computeLogitsGPU(
|
|
|
961
913
|
if (inputBufferOwned && inputBuffer) releaseBuffer(inputBuffer);
|
|
962
914
|
if (normInputOwned && normInputTensor) releaseBuffer(normInputTensor.buffer);
|
|
963
915
|
if (lmHeadInputOwned && lmHeadInputTensor) releaseBuffer(lmHeadInputTensor.buffer);
|
|
916
|
+
if (logitInputOwned && logitInputTensor) releaseBuffer(logitInputTensor.buffer);
|
|
964
917
|
if (finalNormTensor) releaseBuffer(finalNormTensor.buffer);
|
|
965
918
|
if (normedTensor) releaseBuffer(normedTensor.buffer);
|
|
966
919
|
if (normWeightBufferOwned && normWeightBuffer) releaseBuffer(normWeightBuffer);
|
|
@@ -999,13 +952,9 @@ async function recordLogitsTailGPU(
|
|
|
999
952
|
|
|
1000
953
|
let normWeightBuffer;
|
|
1001
954
|
let normWeightOwned = false;
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
normWeightBuffer = acquireBuffer( (finalNorm).byteLength, undefined, 'final_norm_w');
|
|
1006
|
-
recorder.device.queue.writeBuffer(normWeightBuffer, 0, (finalNorm));
|
|
1007
|
-
normWeightOwned = true;
|
|
1008
|
-
}
|
|
955
|
+
const resolvedFinalNorm = resolveFinalNormGpuBuffer(finalNorm, recorder.device.queue, 'final_norm_w');
|
|
956
|
+
normWeightBuffer = resolvedFinalNorm.buffer;
|
|
957
|
+
normWeightOwned = resolvedFinalNorm.owned;
|
|
1009
958
|
|
|
1010
959
|
|
|
1011
960
|
const inputDtype = activationDtype;
|
|
@@ -1080,20 +1029,29 @@ async function recordLogitsTailGPU(
|
|
|
1080
1029
|
operatorDiagnostics,
|
|
1081
1030
|
dtype: finalNormTensor.dtype,
|
|
1082
1031
|
});
|
|
1032
|
+
const logitInputScale = resolveLogitInputScale(config);
|
|
1033
|
+
let logitInputTensor = finalNormTensor;
|
|
1034
|
+
let logitInputOwned = false;
|
|
1035
|
+
if (logitInputScale !== 1) {
|
|
1036
|
+
logitInputTensor = await recordScale(recorder, finalNormTensor, logitInputScale, {
|
|
1037
|
+
count: numTokens * hiddenSize,
|
|
1038
|
+
});
|
|
1039
|
+
logitInputOwned = true;
|
|
1040
|
+
}
|
|
1083
1041
|
const lmHeadRole = resolveLmHeadMatmulRole(phase);
|
|
1084
1042
|
const lmHeadInputDtype = forceStableF32Logits
|
|
1085
|
-
?
|
|
1086
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
1043
|
+
? logitInputTensor.dtype
|
|
1044
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'inputDtype');
|
|
1087
1045
|
const lmHeadOutputDtype = forceStableF32Logits
|
|
1088
|
-
?
|
|
1089
|
-
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath,
|
|
1090
|
-
const lmHeadInputTensor = lmHeadInputDtype !==
|
|
1091
|
-
? await coerceTensorDtype(
|
|
1046
|
+
? logitInputTensor.dtype
|
|
1047
|
+
: resolveMatmulStepDtype(lmHeadRole, phase, stableKernelPath, logitInputTensor.dtype, 'outputDtype');
|
|
1048
|
+
const lmHeadInputTensor = lmHeadInputDtype !== logitInputTensor.dtype
|
|
1049
|
+
? await coerceTensorDtype(logitInputTensor, lmHeadInputDtype, recorder, {
|
|
1092
1050
|
executionPolicies: config.executionPolicies ?? null,
|
|
1093
1051
|
op: 'lm_head',
|
|
1094
1052
|
transitionDeclaredBy: 'step_precision',
|
|
1095
1053
|
})
|
|
1096
|
-
:
|
|
1054
|
+
: logitInputTensor;
|
|
1097
1055
|
|
|
1098
1056
|
// Get LM head buffer
|
|
1099
1057
|
|
|
@@ -1119,6 +1077,8 @@ async function recordLogitsTailGPU(
|
|
|
1119
1077
|
lmHeadOutputDtype,
|
|
1120
1078
|
normedTensor,
|
|
1121
1079
|
finalNormTensor,
|
|
1080
|
+
logitInputTensor,
|
|
1081
|
+
logitInputOwned,
|
|
1122
1082
|
normInputTensor,
|
|
1123
1083
|
normInputOwned,
|
|
1124
1084
|
normWeightBuffer,
|
|
@@ -1142,7 +1102,10 @@ function trackRecordedLogitsTail(recorder, tail) {
|
|
|
1142
1102
|
trackTempBufferOnce(tail.normedTensor.buffer);
|
|
1143
1103
|
}
|
|
1144
1104
|
trackTempBufferOnce(tail.finalNormTensor.buffer);
|
|
1145
|
-
if (tail.
|
|
1105
|
+
if (tail.logitInputOwned) {
|
|
1106
|
+
trackTempBufferOnce(tail.logitInputTensor.buffer);
|
|
1107
|
+
}
|
|
1108
|
+
if (tail.lmHeadInputTensor !== tail.logitInputTensor) {
|
|
1146
1109
|
trackTempBufferOnce(tail.lmHeadInputTensor.buffer);
|
|
1147
1110
|
}
|
|
1148
1111
|
if (tail.normWeightOwned) {
|
|
@@ -1213,7 +1176,7 @@ export async function recordGreedyLmHeadArgmaxGPU(
|
|
|
1213
1176
|
config,
|
|
1214
1177
|
operatorDiagnostics
|
|
1215
1178
|
);
|
|
1216
|
-
const outputBuffer = await
|
|
1179
|
+
const outputBuffer = await recordLmHeadArgmax(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
|
|
1217
1180
|
vocabSize: tail.matmulVocabSize,
|
|
1218
1181
|
hiddenSize: tail.hiddenSize,
|
|
1219
1182
|
padTokenId: options.padTokenId,
|