doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { log } from '../../../debug/index.js';
|
|
2
2
|
import { mergeConfig, dumpConfigSources } from '../../../config/merge.js';
|
|
3
|
+
import { validateModelOverrides } from '../../../config/param-validator.js';
|
|
3
4
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
4
5
|
import {
|
|
5
6
|
PER_LAYER_INPUT_MATERIALIZATION_MODES,
|
|
@@ -222,10 +223,22 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
222
223
|
`"${String(tokenChunk)}"; expected null or a positive integer.`
|
|
223
224
|
);
|
|
224
225
|
}
|
|
226
|
+
const skipEmbeddingKVCacheWrites = session?.skipEmbeddingKVCacheWrites;
|
|
227
|
+
if (
|
|
228
|
+
skipEmbeddingKVCacheWrites !== undefined
|
|
229
|
+
&& skipEmbeddingKVCacheWrites !== null
|
|
230
|
+
&& typeof skipEmbeddingKVCacheWrites !== 'boolean'
|
|
231
|
+
) {
|
|
232
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.skipEmbeddingKVCacheWrites "${String(skipEmbeddingKVCacheWrites)}"; expected boolean.`);
|
|
233
|
+
}
|
|
225
234
|
const flash = session?.useFlashPrefillAttention;
|
|
226
235
|
if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
|
|
227
236
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
|
|
228
237
|
}
|
|
238
|
+
const largeBatchF16F32FusedGateUp = session?.useLargeBatchF16F32FusedGateUp;
|
|
239
|
+
if (largeBatchF16F32FusedGateUp !== undefined && largeBatchF16F32FusedGateUp !== null && typeof largeBatchF16F32FusedGateUp !== 'boolean') {
|
|
240
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLargeBatchF16F32FusedGateUp "${String(largeBatchF16F32FusedGateUp)}"; expected boolean.`);
|
|
241
|
+
}
|
|
229
242
|
const wide = session?.useWideTileQ4KPrefill;
|
|
230
243
|
if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
|
|
231
244
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
|
|
@@ -242,6 +255,30 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
242
255
|
if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
|
|
243
256
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
|
|
244
257
|
}
|
|
258
|
+
const postAttnNormFusedGateUp = session?.usePostAttnNormFusedGateUp;
|
|
259
|
+
if (postAttnNormFusedGateUp !== undefined && postAttnNormFusedGateUp !== null && typeof postAttnNormFusedGateUp !== 'boolean') {
|
|
260
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostAttnNormFusedGateUp "${String(postAttnNormFusedGateUp)}"; expected boolean.`);
|
|
261
|
+
}
|
|
262
|
+
const linearAttentionABProjectionFusion = session?.useLinearAttentionABProjectionFusion;
|
|
263
|
+
if (linearAttentionABProjectionFusion !== undefined && linearAttentionABProjectionFusion !== null && typeof linearAttentionABProjectionFusion !== 'boolean') {
|
|
264
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionABProjectionFusion "${String(linearAttentionABProjectionFusion)}"; expected boolean.`);
|
|
265
|
+
}
|
|
266
|
+
const linearAttentionQKVZProjectionFusion = session?.useLinearAttentionQKVZProjectionFusion;
|
|
267
|
+
if (linearAttentionQKVZProjectionFusion !== undefined && linearAttentionQKVZProjectionFusion !== null && typeof linearAttentionQKVZProjectionFusion !== 'boolean') {
|
|
268
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionQKVZProjectionFusion "${String(linearAttentionQKVZProjectionFusion)}"; expected boolean.`);
|
|
269
|
+
}
|
|
270
|
+
const linearAttentionFusedDecodeCore = session?.useLinearAttentionFusedDecodeCore;
|
|
271
|
+
if (linearAttentionFusedDecodeCore !== undefined && linearAttentionFusedDecodeCore !== null && typeof linearAttentionFusedDecodeCore !== 'boolean') {
|
|
272
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionFusedDecodeCore "${String(linearAttentionFusedDecodeCore)}"; expected boolean.`);
|
|
273
|
+
}
|
|
274
|
+
const wideTileResidualFusion = session?.useWideTileResidualFusion;
|
|
275
|
+
if (wideTileResidualFusion !== undefined && wideTileResidualFusion !== null && typeof wideTileResidualFusion !== 'boolean') {
|
|
276
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileResidualFusion "${String(wideTileResidualFusion)}"; expected boolean.`);
|
|
277
|
+
}
|
|
278
|
+
const fusedRmsnormWideTile = session?.useFusedRmsnormWideTile;
|
|
279
|
+
if (fusedRmsnormWideTile !== undefined && fusedRmsnormWideTile !== null && typeof fusedRmsnormWideTile !== 'boolean') {
|
|
280
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedRmsnormWideTile "${String(fusedRmsnormWideTile)}"; expected boolean.`);
|
|
281
|
+
}
|
|
245
282
|
const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
|
|
246
283
|
if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
|
|
247
284
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
|
|
@@ -254,17 +291,81 @@ function resolveSessionSettings(inferenceConfig, modelId) {
|
|
|
254
291
|
if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
|
|
255
292
|
throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
|
|
256
293
|
}
|
|
294
|
+
const lmHeadArgmaxQ4K = session?.lmHeadArgmaxQ4K;
|
|
295
|
+
if (lmHeadArgmaxQ4K !== undefined && lmHeadArgmaxQ4K !== null) {
|
|
296
|
+
if (typeof lmHeadArgmaxQ4K !== 'object' || Array.isArray(lmHeadArgmaxQ4K)) {
|
|
297
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K; expected object or null.`);
|
|
298
|
+
}
|
|
299
|
+
const fullBlockFastPath = lmHeadArgmaxQ4K.useFullBlockFastPath;
|
|
300
|
+
if (fullBlockFastPath !== undefined && typeof fullBlockFastPath !== 'boolean') {
|
|
301
|
+
throw new Error(
|
|
302
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.useFullBlockFastPath ` +
|
|
303
|
+
`"${String(fullBlockFastPath)}"; expected boolean.`
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
const colsPerWorkgroup = lmHeadArgmaxQ4K.colsPerWorkgroup;
|
|
307
|
+
if (colsPerWorkgroup !== undefined && (!Number.isInteger(colsPerWorkgroup) || colsPerWorkgroup <= 0)) {
|
|
308
|
+
throw new Error(
|
|
309
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.colsPerWorkgroup ` +
|
|
310
|
+
`"${String(colsPerWorkgroup)}"; expected positive integer.`
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
const threadsPerCol = lmHeadArgmaxQ4K.threadsPerCol;
|
|
314
|
+
if (threadsPerCol !== undefined && (!Number.isInteger(threadsPerCol) || threadsPerCol <= 0)) {
|
|
315
|
+
throw new Error(
|
|
316
|
+
`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.threadsPerCol ` +
|
|
317
|
+
`"${String(threadsPerCol)}"; expected positive integer.`
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
const attentionDecodeOnline = session?.attentionDecodeOnline;
|
|
322
|
+
if (attentionDecodeOnline !== undefined && attentionDecodeOnline !== null) {
|
|
323
|
+
if (typeof attentionDecodeOnline !== 'object' || Array.isArray(attentionDecodeOnline)) {
|
|
324
|
+
throw new Error(`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline; expected object or null.`);
|
|
325
|
+
}
|
|
326
|
+
const workgroupSize = attentionDecodeOnline.workgroupSize;
|
|
327
|
+
if (workgroupSize !== undefined && workgroupSize !== 128 && workgroupSize !== 256) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.workgroupSize ` +
|
|
330
|
+
`"${String(workgroupSize)}"; expected 128 or 256.`
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
const directKv = attentionDecodeOnline.useDirectContiguousKVLayout;
|
|
334
|
+
if (directKv !== undefined && typeof directKv !== 'boolean') {
|
|
335
|
+
throw new Error(
|
|
336
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useDirectContiguousKVLayout ` +
|
|
337
|
+
`"${String(directKv)}"; expected boolean.`
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
const outputGateFusion = attentionDecodeOnline.useOutputGateFusion;
|
|
341
|
+
if (outputGateFusion !== undefined && typeof outputGateFusion !== 'boolean') {
|
|
342
|
+
throw new Error(
|
|
343
|
+
`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useOutputGateFusion ` +
|
|
344
|
+
`"${String(outputGateFusion)}"; expected boolean.`
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
257
348
|
return {
|
|
258
349
|
prefillChunkSubmitMode: submit ?? null,
|
|
259
350
|
prefillTokenChunkSize: tokenChunk ?? null,
|
|
351
|
+
skipEmbeddingKVCacheWrites: skipEmbeddingKVCacheWrites ?? null,
|
|
260
352
|
useFlashPrefillAttention: flash ?? null,
|
|
353
|
+
useLargeBatchF16F32FusedGateUp: largeBatchF16F32FusedGateUp ?? null,
|
|
261
354
|
useWideTileQ4KPrefill: wide ?? null,
|
|
262
355
|
useWideTileQ4KDecode: wideDecode ?? null,
|
|
263
356
|
useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
|
|
264
357
|
usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
|
|
358
|
+
usePostAttnNormFusedGateUp: postAttnNormFusedGateUp ?? null,
|
|
359
|
+
useLinearAttentionABProjectionFusion: linearAttentionABProjectionFusion ?? null,
|
|
360
|
+
useLinearAttentionQKVZProjectionFusion: linearAttentionQKVZProjectionFusion ?? null,
|
|
361
|
+
useLinearAttentionFusedDecodeCore: linearAttentionFusedDecodeCore ?? null,
|
|
362
|
+
useWideTileResidualFusion: wideTileResidualFusion ?? null,
|
|
363
|
+
useFusedRmsnormWideTile: fusedRmsnormWideTile ?? null,
|
|
265
364
|
useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
|
|
266
365
|
useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
|
|
267
366
|
retainQ4KMaterialization: retain ?? null,
|
|
367
|
+
lmHeadArgmaxQ4K: lmHeadArgmaxQ4K ?? null,
|
|
368
|
+
attentionDecodeOnline: attentionDecodeOnline ?? null,
|
|
268
369
|
};
|
|
269
370
|
}
|
|
270
371
|
|
|
@@ -376,90 +477,8 @@ function normalizeFfnTensorShape(value) {
|
|
|
376
477
|
return [Math.trunc(rows), Math.trunc(cols)];
|
|
377
478
|
}
|
|
378
479
|
|
|
379
|
-
function
|
|
380
|
-
|
|
381
|
-
return lower.includes('.experts.') || lower.includes('.expert.') || lower.includes('block_sparse_moe');
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
function inferLfm2IntermediateSizeFromManifest(manifest) {
|
|
385
|
-
const tensors = manifest?.tensors;
|
|
386
|
-
if (!tensors || typeof tensors !== 'object') return null;
|
|
387
|
-
const candidates = [];
|
|
388
|
-
for (const [name, entry] of Object.entries(tensors)) {
|
|
389
|
-
if (!name || isExpertTensorName(name)) continue;
|
|
390
|
-
const shape = normalizeFfnTensorShape(entry?.shape);
|
|
391
|
-
if (!shape) continue;
|
|
392
|
-
const lower = name.toLowerCase();
|
|
393
|
-
if (
|
|
394
|
-
lower.endsWith('.feed_forward.w1.weight')
|
|
395
|
-
|| lower.endsWith('.feed_forward.w3.weight')
|
|
396
|
-
|| lower.endsWith('.ffn_gate.weight')
|
|
397
|
-
|| lower.endsWith('.ffn_up.weight')
|
|
398
|
-
|| lower.endsWith('.ffn.gate_proj.weight')
|
|
399
|
-
|| lower.endsWith('.ffn.up_proj.weight')
|
|
400
|
-
|| lower.endsWith('.mlp.gate_proj.weight')
|
|
401
|
-
|| lower.endsWith('.mlp.up_proj.weight')
|
|
402
|
-
) {
|
|
403
|
-
candidates.push(shape[0]);
|
|
404
|
-
continue;
|
|
405
|
-
}
|
|
406
|
-
if (
|
|
407
|
-
lower.endsWith('.feed_forward.w2.weight')
|
|
408
|
-
|| lower.endsWith('.ffn_down.weight')
|
|
409
|
-
|| lower.endsWith('.ffn.down_proj.weight')
|
|
410
|
-
|| lower.endsWith('.mlp.down_proj.weight')
|
|
411
|
-
) {
|
|
412
|
-
candidates.push(shape[1]);
|
|
413
|
-
continue;
|
|
414
|
-
}
|
|
415
|
-
if (
|
|
416
|
-
lower.endsWith('.feed_forward.w1_w3.weight')
|
|
417
|
-
|| lower.endsWith('.ffn_gate_up.weight')
|
|
418
|
-
|| lower.endsWith('.ffn.gate_up_proj.weight')
|
|
419
|
-
|| lower.endsWith('.mlp.gate_up_proj.weight')
|
|
420
|
-
) {
|
|
421
|
-
if (shape[0] % 2 === 0) {
|
|
422
|
-
candidates.push(Math.trunc(shape[0] / 2));
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
if (candidates.length === 0) return null;
|
|
427
|
-
const counts = new Map();
|
|
428
|
-
for (const value of candidates) {
|
|
429
|
-
counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
430
|
-
}
|
|
431
|
-
const result = [...counts.entries()]
|
|
432
|
-
.sort((a, b) => {
|
|
433
|
-
if (b[1] !== a[1]) return b[1] - a[1];
|
|
434
|
-
return a[0] - b[0];
|
|
435
|
-
})[0]?.[0] ?? null;
|
|
436
|
-
if (result != null && (!Number.isInteger(result) || result <= 0)) {
|
|
437
|
-
log.warn(
|
|
438
|
-
'Config',
|
|
439
|
-
`inferLfm2IntermediateSizeFromManifest: inferred intermediateSize ${result} is not a positive integer, discarding`
|
|
440
|
-
);
|
|
441
|
-
return null;
|
|
442
|
-
}
|
|
443
|
-
return result;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
function resolveIntermediateSizeForRuntime(manifest, inf, arch, modelId) {
|
|
447
|
-
const fromArch = arch?.intermediateSize;
|
|
448
|
-
if (typeof fromArch !== 'number' || !Number.isFinite(fromArch) || fromArch <= 0) {
|
|
449
|
-
return fromArch;
|
|
450
|
-
}
|
|
451
|
-
const normalizedModelId = String(modelId ?? manifest?.modelId ?? '').toLowerCase();
|
|
452
|
-
if (!normalizedModelId.includes('lfm2')) {
|
|
453
|
-
return fromArch;
|
|
454
|
-
}
|
|
455
|
-
const inferred = inferLfm2IntermediateSizeFromManifest(manifest);
|
|
456
|
-
if (inferred == null || inferred === fromArch) {
|
|
457
|
-
return fromArch;
|
|
458
|
-
}
|
|
459
|
-
throw new Error(
|
|
460
|
-
`Manifest "${modelId}" has intermediateSize=${fromArch}, but FFN tensors imply ${inferred}. ` +
|
|
461
|
-
'Re-convert the model so manifest architecture matches the weights.'
|
|
462
|
-
);
|
|
480
|
+
function resolveIntermediateSizeForRuntime(arch) {
|
|
481
|
+
return arch?.intermediateSize;
|
|
463
482
|
}
|
|
464
483
|
|
|
465
484
|
function buildPerLayerIntermediateSizes({
|
|
@@ -555,6 +574,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
555
574
|
`${genericPrefix}.mlp.gate_proj.weight`,
|
|
556
575
|
`${languagePrefix}.ffn.gate_proj.weight`,
|
|
557
576
|
`${languagePrefix}.ffn_gate.weight`,
|
|
577
|
+
`${languagePrefix}.feed_forward.w1.weight`,
|
|
578
|
+
`${genericPrefix}.feed_forward.w1.weight`,
|
|
558
579
|
`layers.${layerIdx}.feed_forward.w1.weight`,
|
|
559
580
|
]);
|
|
560
581
|
const upShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -562,6 +583,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
562
583
|
`${genericPrefix}.mlp.up_proj.weight`,
|
|
563
584
|
`${languagePrefix}.ffn.up_proj.weight`,
|
|
564
585
|
`${languagePrefix}.ffn_up.weight`,
|
|
586
|
+
`${languagePrefix}.feed_forward.w3.weight`,
|
|
587
|
+
`${genericPrefix}.feed_forward.w3.weight`,
|
|
565
588
|
`layers.${layerIdx}.feed_forward.w3.weight`,
|
|
566
589
|
]);
|
|
567
590
|
const downShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -569,6 +592,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
569
592
|
`${genericPrefix}.mlp.down_proj.weight`,
|
|
570
593
|
`${languagePrefix}.ffn.down_proj.weight`,
|
|
571
594
|
`${languagePrefix}.ffn_down.weight`,
|
|
595
|
+
`${languagePrefix}.feed_forward.w2.weight`,
|
|
596
|
+
`${genericPrefix}.feed_forward.w2.weight`,
|
|
572
597
|
`layers.${layerIdx}.feed_forward.w2.weight`,
|
|
573
598
|
]);
|
|
574
599
|
const gateUpShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -576,6 +601,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
576
601
|
`${genericPrefix}.mlp.gate_up_proj.weight`,
|
|
577
602
|
`${languagePrefix}.ffn.gate_up_proj.weight`,
|
|
578
603
|
`${languagePrefix}.ffn_gate_up.weight`,
|
|
604
|
+
`${languagePrefix}.feed_forward.w1_w3.weight`,
|
|
605
|
+
`${genericPrefix}.feed_forward.w1_w3.weight`,
|
|
579
606
|
`layers.${layerIdx}.feed_forward.w1_w3.weight`,
|
|
580
607
|
]);
|
|
581
608
|
|
|
@@ -752,7 +779,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
752
779
|
if (inf.ffn.gatedActivation == null) {
|
|
753
780
|
errors.push('ffn.gatedActivation is required');
|
|
754
781
|
}
|
|
755
|
-
if (inf.ffn.branchMode
|
|
782
|
+
if (inf.ffn.branchMode == null) {
|
|
783
|
+
errors.push('ffn.branchMode is required');
|
|
784
|
+
} else {
|
|
756
785
|
const normalizedBranchMode = typeof inf.ffn.branchMode === 'string'
|
|
757
786
|
? inf.ffn.branchMode.trim().toLowerCase()
|
|
758
787
|
: '';
|
|
@@ -840,6 +869,26 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
840
869
|
if (inf.output.scaleEmbeddings == null) {
|
|
841
870
|
errors.push('output.scaleEmbeddings is required');
|
|
842
871
|
}
|
|
872
|
+
if (inf.output.embeddingScale === undefined) {
|
|
873
|
+
errors.push('output.embeddingScale must be explicitly set (null for scaleEmbeddings semantics, or number)');
|
|
874
|
+
} else {
|
|
875
|
+
const embeddingScale = inf.output.embeddingScale;
|
|
876
|
+
if (embeddingScale !== null && (typeof embeddingScale !== 'number' || !Number.isFinite(embeddingScale) || embeddingScale <= 0)) {
|
|
877
|
+
errors.push('output.embeddingScale must be a positive finite number or null');
|
|
878
|
+
}
|
|
879
|
+
if (embeddingScale !== null && inf.output.scaleEmbeddings === true) {
|
|
880
|
+
errors.push('output.embeddingScale cannot be set when output.scaleEmbeddings is true');
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
if (inf.output.logitInputScale == null) {
|
|
884
|
+
errors.push('output.logitInputScale is required');
|
|
885
|
+
} else if (
|
|
886
|
+
typeof inf.output.logitInputScale !== 'number'
|
|
887
|
+
|| !Number.isFinite(inf.output.logitInputScale)
|
|
888
|
+
|| inf.output.logitInputScale <= 0
|
|
889
|
+
) {
|
|
890
|
+
errors.push('output.logitInputScale must be a positive finite number');
|
|
891
|
+
}
|
|
843
892
|
if (inf.output.embeddingTranspose == null) {
|
|
844
893
|
errors.push('output.embeddingTranspose is required');
|
|
845
894
|
}
|
|
@@ -909,6 +958,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
909
958
|
if (inf.layerPattern?.offset === undefined) {
|
|
910
959
|
errors.push('layerPattern.offset must be explicitly set (null if not applicable)');
|
|
911
960
|
}
|
|
961
|
+
if (inf.layerPattern?.residualBranchScale == null) {
|
|
962
|
+
errors.push('layerPattern.residualBranchScale is required');
|
|
963
|
+
} else if (
|
|
964
|
+
typeof inf.layerPattern.residualBranchScale !== 'number'
|
|
965
|
+
|| !Number.isFinite(inf.layerPattern.residualBranchScale)
|
|
966
|
+
|| inf.layerPattern.residualBranchScale <= 0
|
|
967
|
+
) {
|
|
968
|
+
errors.push('layerPattern.residualBranchScale must be a positive finite number');
|
|
969
|
+
}
|
|
912
970
|
if (inf.layerPattern?.type === 'custom' && inf.layerPattern?.layerTypes === undefined) {
|
|
913
971
|
errors.push('layerPattern.layerTypes must be explicitly set for custom patterns');
|
|
914
972
|
}
|
|
@@ -931,6 +989,36 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
931
989
|
if (inf.rope.yarnOriginalMaxPos === undefined) {
|
|
932
990
|
errors.push('rope.yarnOriginalMaxPos must be explicitly set (null if not YARN)');
|
|
933
991
|
}
|
|
992
|
+
if (inf.rope.longropeShortFactor === undefined) {
|
|
993
|
+
errors.push('rope.longropeShortFactor must be explicitly set (null if not LongRoPE)');
|
|
994
|
+
}
|
|
995
|
+
if (inf.rope.longropeLongFactor === undefined) {
|
|
996
|
+
errors.push('rope.longropeLongFactor must be explicitly set (null if not LongRoPE)');
|
|
997
|
+
}
|
|
998
|
+
if (inf.rope.longropeOriginalMaxPos === undefined) {
|
|
999
|
+
errors.push('rope.longropeOriginalMaxPos must be explicitly set (null if not LongRoPE)');
|
|
1000
|
+
}
|
|
1001
|
+
if (inf.rope.ropeScalingType === 'longrope') {
|
|
1002
|
+
if (!Array.isArray(inf.rope.longropeShortFactor) || inf.rope.longropeShortFactor.length === 0) {
|
|
1003
|
+
errors.push('rope.longropeShortFactor must be a non-empty number array for LongRoPE');
|
|
1004
|
+
}
|
|
1005
|
+
if (!Array.isArray(inf.rope.longropeLongFactor) || inf.rope.longropeLongFactor.length === 0) {
|
|
1006
|
+
errors.push('rope.longropeLongFactor must be a non-empty number array for LongRoPE');
|
|
1007
|
+
}
|
|
1008
|
+
if (
|
|
1009
|
+
typeof inf.rope.longropeOriginalMaxPos !== 'number'
|
|
1010
|
+
|| !Number.isFinite(inf.rope.longropeOriginalMaxPos)
|
|
1011
|
+
|| inf.rope.longropeOriginalMaxPos <= 0
|
|
1012
|
+
) {
|
|
1013
|
+
errors.push('rope.longropeOriginalMaxPos must be a positive finite number for LongRoPE');
|
|
1014
|
+
}
|
|
1015
|
+
} else if (
|
|
1016
|
+
inf.rope.longropeShortFactor !== null
|
|
1017
|
+
|| inf.rope.longropeLongFactor !== null
|
|
1018
|
+
|| inf.rope.longropeOriginalMaxPos !== null
|
|
1019
|
+
) {
|
|
1020
|
+
errors.push('rope LongRoPE fields must be null unless rope.ropeScalingType is "longrope"');
|
|
1021
|
+
}
|
|
934
1022
|
if (inf.rope.ropeLocalYarnBetaFast === undefined) {
|
|
935
1023
|
errors.push('rope.ropeLocalYarnBetaFast must be explicitly set (null if not local YARN)');
|
|
936
1024
|
}
|
|
@@ -1228,6 +1316,25 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1228
1316
|
}
|
|
1229
1317
|
return number;
|
|
1230
1318
|
};
|
|
1319
|
+
const resolveRequiredFiniteNumber = (value, label) => {
|
|
1320
|
+
const number = Number(value);
|
|
1321
|
+
if (!Number.isFinite(number)) {
|
|
1322
|
+
throw new Error(
|
|
1323
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1324
|
+
'Expected a finite number.'
|
|
1325
|
+
);
|
|
1326
|
+
}
|
|
1327
|
+
return number;
|
|
1328
|
+
};
|
|
1329
|
+
const resolveRequiredString = (value, label) => {
|
|
1330
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
1331
|
+
throw new Error(
|
|
1332
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1333
|
+
'Expected a non-empty string.'
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1336
|
+
return value.trim();
|
|
1337
|
+
};
|
|
1231
1338
|
const audioArchitecture = String(ac.audio_architecture ?? '').trim();
|
|
1232
1339
|
if (audioArchitecture !== 'gemma4') {
|
|
1233
1340
|
throw new Error(
|
|
@@ -1240,9 +1347,7 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1240
1347
|
const isEncoderFree = (depth === 0);
|
|
1241
1348
|
|
|
1242
1349
|
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1243
|
-
const numAttentionHeads =
|
|
1244
|
-
? (ac.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads') : 1)
|
|
1245
|
-
: resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1350
|
+
const numAttentionHeads = resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1246
1351
|
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1247
1352
|
|
|
1248
1353
|
if (!isEncoderFree) {
|
|
@@ -1259,29 +1364,17 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1259
1364
|
hiddenSize,
|
|
1260
1365
|
numAttentionHeads,
|
|
1261
1366
|
headDim,
|
|
1262
|
-
convKernelSize:
|
|
1263
|
-
|
|
1264
|
-
: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1265
|
-
subsamplingConvChannels: isEncoderFree
|
|
1266
|
-
? (ac.subsampling_conv_channels ? ac.subsampling_conv_channels.map(Number) : [])
|
|
1267
|
-
: ac.subsampling_conv_channels.map(Number),
|
|
1367
|
+
convKernelSize: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1368
|
+
subsamplingConvChannels: ac.subsampling_conv_channels.map(Number),
|
|
1268
1369
|
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1269
|
-
attentionContextLeft:
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
? (ac.attention_logit_cap !== undefined ? resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap') : 1.0)
|
|
1278
|
-
: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1279
|
-
attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
|
|
1280
|
-
residualWeight: isEncoderFree
|
|
1281
|
-
? (ac.residual_weight !== undefined ? resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight') : 1.0)
|
|
1282
|
-
: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1283
|
-
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
|
|
1284
|
-
hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
|
|
1370
|
+
attentionContextLeft: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1371
|
+
attentionContextRight: resolveRequiredNonNegativeInteger(ac.attention_context_right, 'attention_context_right'),
|
|
1372
|
+
attentionChunkSize: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1373
|
+
attentionLogitCap: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1374
|
+
attentionInvalidLogitsValue: resolveRequiredFiniteNumber(ac.attention_invalid_logits_value, 'attention_invalid_logits_value'),
|
|
1375
|
+
residualWeight: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1376
|
+
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps, 'rms_norm_eps'),
|
|
1377
|
+
hiddenAct: resolveRequiredString(ac.hidden_act, 'hidden_act'),
|
|
1285
1378
|
useClippedLinears: ac.use_clipped_linears === true,
|
|
1286
1379
|
audioTokenId: rawConfig?.audio_token_id ?? manifest?.audio_token_id ?? null,
|
|
1287
1380
|
};
|
|
@@ -1364,7 +1457,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1364
1457
|
`Re-convert the model using the latest converter to add manifest.architecture.`
|
|
1365
1458
|
);
|
|
1366
1459
|
}
|
|
1367
|
-
const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(
|
|
1460
|
+
const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(arch);
|
|
1368
1461
|
const archNumHeads = Number(arch.numAttentionHeads ?? arch.numHeads);
|
|
1369
1462
|
const archNumKVHeads = Number(arch.numKeyValueHeads ?? arch.numKVHeads);
|
|
1370
1463
|
const archHeadDim = Number(arch.headDim);
|
|
@@ -1569,6 +1662,24 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1569
1662
|
);
|
|
1570
1663
|
}
|
|
1571
1664
|
}
|
|
1665
|
+
const validateLongropeFactorArray = (value, label) => {
|
|
1666
|
+
if (!Array.isArray(value)) {
|
|
1667
|
+
throw new Error(`Manifest "${merged.modelId}" ${label} must be an array for LongRoPE.`);
|
|
1668
|
+
}
|
|
1669
|
+
if (value.length !== ropeRotaryDim / 2) {
|
|
1670
|
+
throw new Error(
|
|
1671
|
+
`Manifest "${merged.modelId}" ${label} length ${value.length} does not match ` +
|
|
1672
|
+
`resolved RoPE half dim ${ropeRotaryDim / 2}.`
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
1676
|
+
const entry = Number(value[i]);
|
|
1677
|
+
if (!Number.isFinite(entry) || entry <= 0) {
|
|
1678
|
+
throw new Error(`Manifest "${merged.modelId}" ${label}[${i}] must be a positive finite number.`);
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
return value.map((entry) => Number(entry));
|
|
1682
|
+
};
|
|
1572
1683
|
|
|
1573
1684
|
// Build ropeScaling object from manifest values if scaling is enabled
|
|
1574
1685
|
// Include YARN params when present
|
|
@@ -1581,6 +1692,11 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1581
1692
|
...(ropeScalingType === 'yarn' && inf.rope.yarnOriginalMaxPos != null && {
|
|
1582
1693
|
original_max_position_embeddings: inf.rope.yarnOriginalMaxPos
|
|
1583
1694
|
}),
|
|
1695
|
+
...(ropeScalingType === 'longrope' && {
|
|
1696
|
+
short_factor: validateLongropeFactorArray(inf.rope.longropeShortFactor, 'rope.longropeShortFactor'),
|
|
1697
|
+
long_factor: validateLongropeFactorArray(inf.rope.longropeLongFactor, 'rope.longropeLongFactor'),
|
|
1698
|
+
original_max_position_embeddings: inf.rope.longropeOriginalMaxPos,
|
|
1699
|
+
}),
|
|
1584
1700
|
} : null;
|
|
1585
1701
|
const ropeLocalScaling = ropeLocalScalingType ? {
|
|
1586
1702
|
type: ropeLocalScalingType,
|
|
@@ -1681,6 +1797,9 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1681
1797
|
const sessionSettings = resolveSessionSettings(inf, merged.modelId);
|
|
1682
1798
|
const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
|
|
1683
1799
|
const diffusionGemma = inf.diffusionGemma ?? null;
|
|
1800
|
+
const embeddingScale = inf.output.embeddingScale;
|
|
1801
|
+
const logitInputScale = inf.output.logitInputScale;
|
|
1802
|
+
const residualBranchScale = inf.layerPattern.residualBranchScale;
|
|
1684
1803
|
|
|
1685
1804
|
return {
|
|
1686
1805
|
modelType: manifest.modelType,
|
|
@@ -1730,12 +1849,15 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1730
1849
|
preFeedforwardNorm: inf.normalization.preFeedforwardNorm,
|
|
1731
1850
|
postFeedforwardNorm: inf.normalization.postFeedforwardNorm,
|
|
1732
1851
|
scaleEmbeddings: inf.output.scaleEmbeddings,
|
|
1852
|
+
embeddingScale,
|
|
1853
|
+
logitInputScale,
|
|
1854
|
+
residualBranchScale,
|
|
1733
1855
|
useTiedEmbeddings: inf.output.tieWordEmbeddings,
|
|
1734
1856
|
embeddingTranspose: inf.output.embeddingTranspose,
|
|
1735
1857
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1736
1858
|
embeddingPostprocessor,
|
|
1737
1859
|
hiddenActivation,
|
|
1738
|
-
ffnBranchMode: inf.ffn.branchMode
|
|
1860
|
+
ffnBranchMode: inf.ffn.branchMode,
|
|
1739
1861
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1740
1862
|
swigluLimit: inf.ffn.swigluLimit,
|
|
1741
1863
|
stopTokenIds,
|
|
@@ -1775,6 +1897,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1775
1897
|
|
|
1776
1898
|
export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
|
|
1777
1899
|
assertSupportedRuntimeModelType(manifest);
|
|
1900
|
+
validateModelOverrides(runtimeOverrides, 'runtime.inference.modelOverrides');
|
|
1778
1901
|
|
|
1779
1902
|
// Merge manifest inference with runtime overrides
|
|
1780
1903
|
const merged = mergeConfig(
|
|
@@ -63,6 +63,28 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
function resolveEmbeddingScale(config, hiddenSize) {
|
|
67
|
+
const embeddingScale = config.embeddingScale;
|
|
68
|
+
const scaleEmbeddings = config.scaleEmbeddings;
|
|
69
|
+
if (embeddingScale === undefined) {
|
|
70
|
+
throw new Error('[Embed] embeddingScale must be explicitly set (null to use scaleEmbeddings semantics).');
|
|
71
|
+
}
|
|
72
|
+
if (scaleEmbeddings == null) {
|
|
73
|
+
throw new Error('[Embed] scaleEmbeddings is required.');
|
|
74
|
+
}
|
|
75
|
+
if (embeddingScale !== null) {
|
|
76
|
+
const value = Number(embeddingScale);
|
|
77
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
78
|
+
throw new Error(`[Embed] embeddingScale must be a positive finite number or null; got "${String(embeddingScale)}".`);
|
|
79
|
+
}
|
|
80
|
+
if (scaleEmbeddings === true) {
|
|
81
|
+
throw new Error('[Embed] embeddingScale cannot be set when scaleEmbeddings is true.');
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
return scaleEmbeddings === true ? Math.sqrt(hiddenSize) : 1;
|
|
86
|
+
}
|
|
87
|
+
|
|
66
88
|
async function readGpuTokenIdsForCpuEmbeddingGather(tokenIds, numTokens, indexOffset) {
|
|
67
89
|
if (numTokens <= 0) {
|
|
68
90
|
throw new Error('[Embed] numTokens must be provided when tokenIds is a GPUBuffer.');
|
|
@@ -77,6 +99,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
77
99
|
hiddenSize,
|
|
78
100
|
vocabSize,
|
|
79
101
|
scaleEmbeddings,
|
|
102
|
+
embeddingScale,
|
|
80
103
|
debug = false,
|
|
81
104
|
recorder,
|
|
82
105
|
outputBuffer: preAllocatedOutput,
|
|
@@ -91,6 +114,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
91
114
|
embeddingStorageEncoding = null,
|
|
92
115
|
} = config;
|
|
93
116
|
const device = getDevice();
|
|
117
|
+
const resolvedEmbeddingScale = resolveEmbeddingScale({ scaleEmbeddings, embeddingScale }, hiddenSize);
|
|
94
118
|
const tokenBufferInput = isGpuBufferInstance(tokenIds);
|
|
95
119
|
let tokenIdArray = tokenBufferInput ? null : (tokenIds);
|
|
96
120
|
const numTokens = tokenBufferInput
|
|
@@ -193,7 +217,14 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
193
217
|
? cpuEmbeddings
|
|
194
218
|
: null;
|
|
195
219
|
if (rangeBackedSource) {
|
|
196
|
-
const
|
|
220
|
+
const rawSourceDtype = rangeBackedSource.sourceDtype ?? embedBuffer.dtype;
|
|
221
|
+
if (rawSourceDtype == null) {
|
|
222
|
+
throw new Error('[Embed] CPU embedding range source requires sourceDtype or embedding buffer dtype metadata.');
|
|
223
|
+
}
|
|
224
|
+
const sourceDtype = String(rawSourceDtype).toLowerCase();
|
|
225
|
+
if (sourceDtype !== 'f16' && sourceDtype !== 'bf16' && sourceDtype !== 'f32') {
|
|
226
|
+
throw new Error(`[Embed] CPU embedding range source dtype "${sourceDtype}" is unsupported.`);
|
|
227
|
+
}
|
|
197
228
|
const bytesPerElement = sourceDtype === 'f16' || sourceDtype === 'bf16' ? 2 : 4;
|
|
198
229
|
if (!transpose) {
|
|
199
230
|
for (let t = 0; t < numTokens; t++) {
|
|
@@ -256,10 +287,9 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
256
287
|
}
|
|
257
288
|
} // end else (non-preloaded path)
|
|
258
289
|
|
|
259
|
-
if (
|
|
260
|
-
const scaleFactor = Math.sqrt(hiddenSize);
|
|
290
|
+
if (resolvedEmbeddingScale !== 1) {
|
|
261
291
|
for (let i = 0; i < output.length; i++) {
|
|
262
|
-
output[i] *=
|
|
292
|
+
output[i] *= resolvedEmbeddingScale;
|
|
263
293
|
}
|
|
264
294
|
}
|
|
265
295
|
|
|
@@ -375,7 +405,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
375
405
|
}
|
|
376
406
|
}
|
|
377
407
|
|
|
378
|
-
if (
|
|
408
|
+
if (resolvedEmbeddingScale === 1) {
|
|
379
409
|
await runProbes(probeStage, gatherOutput.buffer, {
|
|
380
410
|
numTokens,
|
|
381
411
|
hiddenSize,
|
|
@@ -387,9 +417,6 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
387
417
|
return gatherOutput;
|
|
388
418
|
}
|
|
389
419
|
|
|
390
|
-
// Apply embedding scaling: sqrt(hiddenSize)
|
|
391
|
-
const scaleFactor = Math.sqrt(hiddenSize);
|
|
392
|
-
|
|
393
420
|
// Debug: check raw embedding values before scaling
|
|
394
421
|
if (debug && !recorder) {
|
|
395
422
|
const bytesPerElement = gatherOptions.outputDtype === 'f16' ? 2 : 4;
|
|
@@ -401,7 +428,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
401
428
|
const abs = Math.abs(f32[i]);
|
|
402
429
|
if (abs > maxAbs) maxAbs = abs;
|
|
403
430
|
}
|
|
404
|
-
trace.embed(`RAW (before scale): maxAbs=${maxAbs.toFixed(4)}, scaleFactor=${
|
|
431
|
+
trace.embed(`RAW (before scale): maxAbs=${maxAbs.toFixed(4)}, scaleFactor=${resolvedEmbeddingScale.toFixed(4)}`);
|
|
405
432
|
}
|
|
406
433
|
|
|
407
434
|
const gatheredTensor = createTensor(
|
|
@@ -411,10 +438,10 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
411
438
|
'embed_gather_output'
|
|
412
439
|
);
|
|
413
440
|
const scaledTensor = recorder
|
|
414
|
-
? await recordScale(recorder, gatheredTensor,
|
|
441
|
+
? await recordScale(recorder, gatheredTensor, resolvedEmbeddingScale, {
|
|
415
442
|
count: numTokens * hiddenSize,
|
|
416
443
|
})
|
|
417
|
-
: await runScale(gatheredTensor,
|
|
444
|
+
: await runScale(gatheredTensor, resolvedEmbeddingScale, {
|
|
418
445
|
count: numTokens * hiddenSize,
|
|
419
446
|
});
|
|
420
447
|
const scaledBuffer = scaledTensor.buffer;
|
|
@@ -441,7 +468,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
441
468
|
const abs = Math.abs(f32[i]);
|
|
442
469
|
if (abs > maxAbs) maxAbs = abs;
|
|
443
470
|
}
|
|
444
|
-
trace.embed(`SCALED (after *${
|
|
471
|
+
trace.embed(`SCALED (after *${resolvedEmbeddingScale.toFixed(2)}): maxAbs=${maxAbs.toFixed(4)}, buffer.label=${scaledBuffer.label}, buffer.size=${scaledBuffer.size}`);
|
|
445
472
|
trace.embed(`RETURNING buffer with first8=[${Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ')}]`);
|
|
446
473
|
if (f32.some(x => !Number.isFinite(x))) {
|
|
447
474
|
throw new Error('[Embed] Scaled embedding contains NaN/Inf');
|
|
@@ -21,6 +21,7 @@ export interface StaticExecutionPlan {
|
|
|
21
21
|
defaultMaxTokens: number;
|
|
22
22
|
readbackInterval: number | null;
|
|
23
23
|
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
24
|
+
maxBatchDecodeTokens?: number | null;
|
|
24
25
|
ringTokens: number | null;
|
|
25
26
|
ringStop: number | null;
|
|
26
27
|
ringStaging: number | null;
|
|
@@ -53,6 +54,7 @@ export interface ExecutionSessionPlan {
|
|
|
53
54
|
maxTokens: number;
|
|
54
55
|
readbackInterval: number | null;
|
|
55
56
|
readbackMode: 'sequential' | 'overlapped' | 'auto';
|
|
57
|
+
maxBatchDecodeTokens?: number | null;
|
|
56
58
|
ringTokens: number | null;
|
|
57
59
|
ringStop: number | null;
|
|
58
60
|
ringStaging: number | null;
|