doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -134,6 +134,36 @@ const FINITENESS_RESET_WORDS = new Uint32Array(4);
|
|
|
134
134
|
const tokenizerSuppressionCache = new WeakMap();
|
|
135
135
|
const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
|
|
136
136
|
|
|
137
|
+
function mergeRecordOpLabelCounts(target, source) {
|
|
138
|
+
const merged = target && typeof target === 'object' && !Array.isArray(target)
|
|
139
|
+
? target
|
|
140
|
+
: {};
|
|
141
|
+
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
142
|
+
return merged;
|
|
143
|
+
}
|
|
144
|
+
for (const [label, rawCount] of Object.entries(source)) {
|
|
145
|
+
const count = Number(rawCount);
|
|
146
|
+
if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
|
|
150
|
+
}
|
|
151
|
+
return merged;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function recordPrefillRecorderStats(state, recorder) {
|
|
155
|
+
if (!recorder || typeof recorder.getStats !== 'function') {
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const recordStats = recorder.getStats();
|
|
159
|
+
state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
|
|
160
|
+
state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
|
|
161
|
+
state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
|
|
162
|
+
state.stats.prefillRecordOpLabels,
|
|
163
|
+
recordStats.opLabelCounts
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
137
167
|
export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
|
|
138
168
|
const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
|
|
139
169
|
const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
|
|
@@ -219,6 +249,26 @@ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
|
|
|
219
249
|
return [...suppressed];
|
|
220
250
|
}
|
|
221
251
|
|
|
252
|
+
function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
|
|
253
|
+
if (value == null) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
|
|
257
|
+
throw new Error(`${label} must be an array or typed array.`);
|
|
258
|
+
}
|
|
259
|
+
const tokenIds = Array.from(value, (entry, index) => {
|
|
260
|
+
const tokenId = Number(entry);
|
|
261
|
+
if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
|
|
262
|
+
throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
|
|
263
|
+
}
|
|
264
|
+
return tokenId;
|
|
265
|
+
});
|
|
266
|
+
if (tokenIds.length === 0) {
|
|
267
|
+
throw new Error(`${label} must not be empty.`);
|
|
268
|
+
}
|
|
269
|
+
return tokenIds;
|
|
270
|
+
}
|
|
271
|
+
|
|
222
272
|
async function traceActivationHealth(label, buffer, dtype, elementCount) {
|
|
223
273
|
if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
|
|
224
274
|
return;
|
|
@@ -535,6 +585,11 @@ export class PipelineGenerator {
|
|
|
535
585
|
this.#state.stats.decodeProfileSteps = [];
|
|
536
586
|
this.#state.stats.ttftMs = 0;
|
|
537
587
|
this.#state.stats.decodeTimeMs = 0;
|
|
588
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
589
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
590
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
591
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
592
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
538
593
|
this.#state.stats.decodeRecordMs = 0;
|
|
539
594
|
this.#state.stats.decodeRecordOps = 0;
|
|
540
595
|
this.#state.stats.decodeRecordPasses = 0;
|
|
@@ -667,6 +722,7 @@ export class PipelineGenerator {
|
|
|
667
722
|
|
|
668
723
|
try {
|
|
669
724
|
if (currentRecorder) {
|
|
725
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
670
726
|
await currentRecorder.submitAndWait();
|
|
671
727
|
await recordProfile(currentRecorder);
|
|
672
728
|
} else {
|
|
@@ -703,12 +759,15 @@ export class PipelineGenerator {
|
|
|
703
759
|
|
|
704
760
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
705
761
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
762
|
+
const inputStart = performance.now();
|
|
706
763
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
764
|
+
const inputMs = performance.now() - inputStart;
|
|
707
765
|
if (opts.debug) {
|
|
708
766
|
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
709
767
|
}
|
|
710
768
|
|
|
711
769
|
let logits;
|
|
770
|
+
const prefillStart = performance.now();
|
|
712
771
|
const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
|
|
713
772
|
try {
|
|
714
773
|
logits = await runPrefill();
|
|
@@ -726,7 +785,21 @@ export class PipelineGenerator {
|
|
|
726
785
|
);
|
|
727
786
|
}
|
|
728
787
|
|
|
729
|
-
return {
|
|
788
|
+
return {
|
|
789
|
+
inputIds,
|
|
790
|
+
logits,
|
|
791
|
+
phase: {
|
|
792
|
+
inputMs,
|
|
793
|
+
prefillMs: performance.now() - prefillStart,
|
|
794
|
+
tokens: inputIds.length,
|
|
795
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
796
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
797
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
798
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
799
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
800
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
801
|
+
},
|
|
802
|
+
};
|
|
730
803
|
}
|
|
731
804
|
|
|
732
805
|
async _decodeStepToLogits(currentIds, opts) {
|
|
@@ -785,6 +858,9 @@ export class PipelineGenerator {
|
|
|
785
858
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
786
859
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
787
860
|
this.#state.stats.prefillRecordMs = 0;
|
|
861
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
862
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
863
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
788
864
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
789
865
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
790
866
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1298,6 +1374,7 @@ export class PipelineGenerator {
|
|
|
1298
1374
|
hiddenSize,
|
|
1299
1375
|
vocabSize,
|
|
1300
1376
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
1377
|
+
embeddingScale: config.embeddingScale,
|
|
1301
1378
|
debug: opts.debug,
|
|
1302
1379
|
recorder: null,
|
|
1303
1380
|
transpose: this.#state.embeddingTranspose,
|
|
@@ -1590,6 +1667,7 @@ export class PipelineGenerator {
|
|
|
1590
1667
|
} = prefillResult;
|
|
1591
1668
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1592
1669
|
if (currentRecorder) {
|
|
1670
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1593
1671
|
await currentRecorder.submitAndWait();
|
|
1594
1672
|
await recordProfile(currentRecorder);
|
|
1595
1673
|
} else {
|
|
@@ -1677,6 +1755,7 @@ export class PipelineGenerator {
|
|
|
1677
1755
|
} = prefillResult;
|
|
1678
1756
|
currentHiddenBuffer = prefillResult.currentHiddenBuffer;
|
|
1679
1757
|
if (currentRecorder) {
|
|
1758
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1680
1759
|
await currentRecorder.submitAndWait();
|
|
1681
1760
|
await recordProfile(currentRecorder);
|
|
1682
1761
|
} else {
|
|
@@ -1809,6 +1888,9 @@ export class PipelineGenerator {
|
|
|
1809
1888
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
1810
1889
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
1811
1890
|
this.#state.stats.prefillRecordMs = 0;
|
|
1891
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
1892
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
1893
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
1812
1894
|
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
1813
1895
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
1814
1896
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
@@ -1990,6 +2072,7 @@ export class PipelineGenerator {
|
|
|
1990
2072
|
|
|
1991
2073
|
// Ensure prefill work completes before returning a usable snapshot.
|
|
1992
2074
|
if (currentRecorder) {
|
|
2075
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
1993
2076
|
await currentRecorder.submitAndWait();
|
|
1994
2077
|
await recordProfile(currentRecorder);
|
|
1995
2078
|
} else {
|
|
@@ -2024,20 +2107,27 @@ export class PipelineGenerator {
|
|
|
2024
2107
|
throw new Error('Generation already in progress');
|
|
2025
2108
|
}
|
|
2026
2109
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
2110
|
+
const totalStart = performance.now();
|
|
2027
2111
|
this._resetDecodeRuntimeState();
|
|
2028
2112
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2029
2113
|
this.#state.stats.prefillProfileSteps = [];
|
|
2030
2114
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2031
2115
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
|
+
const inputStart = performance.now();
|
|
2032
2117
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
2118
|
+
const inputMs = performance.now() - inputStart;
|
|
2033
2119
|
if (opts.debug) {
|
|
2034
2120
|
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
2035
2121
|
}
|
|
2036
2122
|
|
|
2037
2123
|
try {
|
|
2038
2124
|
let prefillResult;
|
|
2125
|
+
const prefillStart = performance.now();
|
|
2039
2126
|
try {
|
|
2040
|
-
prefillResult = await this._prefillToHidden(inputIds,
|
|
2127
|
+
prefillResult = await this._prefillToHidden(inputIds, {
|
|
2128
|
+
...opts,
|
|
2129
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2130
|
+
});
|
|
2041
2131
|
} catch (error) {
|
|
2042
2132
|
if (shouldRetryWithFinitenessFallback(error)) {
|
|
2043
2133
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
@@ -2045,13 +2135,17 @@ export class PipelineGenerator {
|
|
|
2045
2135
|
opts,
|
|
2046
2136
|
'prefillWithEmbedding',
|
|
2047
2137
|
1,
|
|
2048
|
-
() => this._prefillToHidden(inputIds,
|
|
2138
|
+
() => this._prefillToHidden(inputIds, {
|
|
2139
|
+
...opts,
|
|
2140
|
+
_embeddingOnly: options.__skipStateSnapshot === true,
|
|
2141
|
+
}),
|
|
2049
2142
|
prefillStartSeqLen
|
|
2050
2143
|
);
|
|
2051
2144
|
} else {
|
|
2052
2145
|
throw error;
|
|
2053
2146
|
}
|
|
2054
2147
|
}
|
|
2148
|
+
const prefillMs = performance.now() - prefillStart;
|
|
2055
2149
|
|
|
2056
2150
|
const {
|
|
2057
2151
|
numTokens,
|
|
@@ -2065,7 +2159,10 @@ export class PipelineGenerator {
|
|
|
2065
2159
|
} = prefillResult;
|
|
2066
2160
|
|
|
2067
2161
|
// Ensure prefill work completes before readback.
|
|
2162
|
+
let submitWaitMs = 0;
|
|
2163
|
+
const submitWaitStart = performance.now();
|
|
2068
2164
|
if (currentRecorder) {
|
|
2165
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
2069
2166
|
await currentRecorder.submitAndWait();
|
|
2070
2167
|
await recordProfile(currentRecorder);
|
|
2071
2168
|
} else {
|
|
@@ -2074,21 +2171,34 @@ export class PipelineGenerator {
|
|
|
2074
2171
|
await device.queue.onSubmittedWorkDone();
|
|
2075
2172
|
}
|
|
2076
2173
|
}
|
|
2174
|
+
submitWaitMs = performance.now() - submitWaitStart;
|
|
2077
2175
|
|
|
2078
2176
|
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
2079
2177
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2080
2178
|
}
|
|
2081
2179
|
|
|
2082
2180
|
let embedding;
|
|
2181
|
+
let hiddenBytes = 0;
|
|
2182
|
+
let readbackMs = 0;
|
|
2183
|
+
let decodeHiddenMs = 0;
|
|
2184
|
+
let finalNormMs = 0;
|
|
2185
|
+
let extractMs = 0;
|
|
2083
2186
|
try {
|
|
2084
2187
|
const hiddenSize = config.hiddenSize;
|
|
2085
|
-
|
|
2188
|
+
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
|
+
const readbackStart = performance.now();
|
|
2086
2190
|
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
2191
|
+
readbackMs = performance.now() - readbackStart;
|
|
2087
2192
|
if (hiddenData.byteLength === 0) {
|
|
2088
2193
|
throw new Error('GPU readback disabled; cannot return embedding');
|
|
2089
2194
|
}
|
|
2195
|
+
const decodeStart = performance.now();
|
|
2090
2196
|
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
2197
|
+
decodeHiddenMs = performance.now() - decodeStart;
|
|
2198
|
+
const finalNormStart = performance.now();
|
|
2091
2199
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
|
+
finalNormMs = performance.now() - finalNormStart;
|
|
2201
|
+
const extractStart = performance.now();
|
|
2092
2202
|
embedding = this._extractEmbeddingFromHidden(
|
|
2093
2203
|
hiddenStates,
|
|
2094
2204
|
numTokens,
|
|
@@ -2097,11 +2207,31 @@ export class PipelineGenerator {
|
|
|
2097
2207
|
finalNormWeights,
|
|
2098
2208
|
config
|
|
2099
2209
|
);
|
|
2210
|
+
extractMs = performance.now() - extractStart;
|
|
2100
2211
|
} finally {
|
|
2101
2212
|
releaseBuffer(currentHiddenBuffer);
|
|
2102
2213
|
}
|
|
2103
2214
|
|
|
2104
2215
|
this.#state.currentSeqLen = startPos + numTokens;
|
|
2216
|
+
const phase = {
|
|
2217
|
+
totalMs: performance.now() - totalStart,
|
|
2218
|
+
inputMs,
|
|
2219
|
+
prefillMs,
|
|
2220
|
+
submitWaitMs,
|
|
2221
|
+
readbackMs,
|
|
2222
|
+
decodeHiddenMs,
|
|
2223
|
+
finalNormMs,
|
|
2224
|
+
extractMs,
|
|
2225
|
+
hiddenBytes,
|
|
2226
|
+
tokens: numTokens,
|
|
2227
|
+
activationDtype,
|
|
2228
|
+
prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
|
|
2229
|
+
prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
|
|
2230
|
+
prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
|
|
2231
|
+
prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
|
|
2232
|
+
prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
|
|
2233
|
+
gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
|
|
2234
|
+
};
|
|
2105
2235
|
|
|
2106
2236
|
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
2107
2237
|
// since the caller will reset immediately after extracting the embedding.
|
|
@@ -2112,6 +2242,7 @@ export class PipelineGenerator {
|
|
|
2112
2242
|
tokens: inputIds,
|
|
2113
2243
|
embedding,
|
|
2114
2244
|
embeddingMode: opts.embeddingMode,
|
|
2245
|
+
phase,
|
|
2115
2246
|
linearAttention: null,
|
|
2116
2247
|
};
|
|
2117
2248
|
}
|
|
@@ -2127,6 +2258,7 @@ export class PipelineGenerator {
|
|
|
2127
2258
|
tokens: inputIds,
|
|
2128
2259
|
embedding,
|
|
2129
2260
|
embeddingMode: opts.embeddingMode,
|
|
2261
|
+
phase,
|
|
2130
2262
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2131
2263
|
};
|
|
2132
2264
|
} finally {
|
|
@@ -2140,12 +2272,13 @@ export class PipelineGenerator {
|
|
|
2140
2272
|
throw new Error('Generation already in progress');
|
|
2141
2273
|
}
|
|
2142
2274
|
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
2275
|
+
const totalStart = performance.now();
|
|
2143
2276
|
this._resetDecodeRuntimeState();
|
|
2144
2277
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2145
2278
|
this.#state.stats.prefillProfileSteps = [];
|
|
2146
2279
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
2147
2280
|
try {
|
|
2148
|
-
const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2281
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2149
2282
|
|
|
2150
2283
|
const snapshot = this.#state.kvCache?.clone();
|
|
2151
2284
|
if (!snapshot) {
|
|
@@ -2157,6 +2290,10 @@ export class PipelineGenerator {
|
|
|
2157
2290
|
seqLen: this.#state.currentSeqLen,
|
|
2158
2291
|
tokens: inputIds,
|
|
2159
2292
|
logits,
|
|
2293
|
+
phase: {
|
|
2294
|
+
...(prefillPhase ?? {}),
|
|
2295
|
+
totalMs: performance.now() - totalStart,
|
|
2296
|
+
},
|
|
2160
2297
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2161
2298
|
};
|
|
2162
2299
|
} finally {
|
|
@@ -2164,6 +2301,120 @@ export class PipelineGenerator {
|
|
|
2164
2301
|
}
|
|
2165
2302
|
}
|
|
2166
2303
|
|
|
2304
|
+
async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
2305
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2306
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2307
|
+
throw new Error('Generation already in progress');
|
|
2308
|
+
}
|
|
2309
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
|
|
2310
|
+
const totalStart = performance.now();
|
|
2311
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2312
|
+
tokenIds,
|
|
2313
|
+
this.#state.modelConfig.vocabSize,
|
|
2314
|
+
'prefillWithTokenLogits.tokenIds'
|
|
2315
|
+
);
|
|
2316
|
+
this._resetDecodeRuntimeState();
|
|
2317
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2318
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2319
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2320
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2321
|
+
try {
|
|
2322
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
|
|
2323
|
+
const logitsByTokenId = {};
|
|
2324
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2325
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2326
|
+
}
|
|
2327
|
+
return {
|
|
2328
|
+
seqLen: this.#state.currentSeqLen,
|
|
2329
|
+
tokens: inputIds,
|
|
2330
|
+
tokenIds: selectedTokenIds,
|
|
2331
|
+
logits,
|
|
2332
|
+
logitsByTokenId,
|
|
2333
|
+
phase: {
|
|
2334
|
+
...(prefillPhase ?? {}),
|
|
2335
|
+
totalMs: performance.now() - totalStart,
|
|
2336
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2337
|
+
},
|
|
2338
|
+
};
|
|
2339
|
+
} finally {
|
|
2340
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
2345
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
2346
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
2347
|
+
throw new Error('Generation already in progress');
|
|
2348
|
+
}
|
|
2349
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
|
|
2350
|
+
const totalStart = performance.now();
|
|
2351
|
+
if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
|
|
2352
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
|
|
2353
|
+
}
|
|
2354
|
+
if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
|
|
2355
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
|
|
2356
|
+
}
|
|
2357
|
+
if (!Array.isArray(prefix.tokens)) {
|
|
2358
|
+
throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
|
|
2359
|
+
}
|
|
2360
|
+
const selectedTokenIds = normalizeSelectedLogitTokenIds(
|
|
2361
|
+
tokenIds,
|
|
2362
|
+
this.#state.modelConfig.vocabSize,
|
|
2363
|
+
'prefillWithTokenLogitsFromKV.tokenIds'
|
|
2364
|
+
);
|
|
2365
|
+
|
|
2366
|
+
this.#state.kvCache = prefix.cache.clone();
|
|
2367
|
+
if (this.#state.useGPU && this.#state.kvCache) {
|
|
2368
|
+
const device = getDevice();
|
|
2369
|
+
if (device) {
|
|
2370
|
+
this.#state.kvCache.setGPUContext({ device });
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
if (
|
|
2374
|
+
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
2375
|
+
&& prefix.linearAttention == null
|
|
2376
|
+
) {
|
|
2377
|
+
throw new Error(
|
|
2378
|
+
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
2379
|
+
'Regenerate the prefix snapshot using the current runtime.'
|
|
2380
|
+
);
|
|
2381
|
+
}
|
|
2382
|
+
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
2383
|
+
this.#state.linearAttentionRuntime,
|
|
2384
|
+
prefix.linearAttention ?? null
|
|
2385
|
+
);
|
|
2386
|
+
this.#state.currentSeqLen = prefix.seqLen;
|
|
2387
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2388
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
2389
|
+
this.#state.decodeRing?.reset();
|
|
2390
|
+
|
|
2391
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
2392
|
+
opts._selectedLogitTokenIds = selectedTokenIds;
|
|
2393
|
+
try {
|
|
2394
|
+
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
|
|
2395
|
+
const logitsByTokenId = {};
|
|
2396
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
2397
|
+
logitsByTokenId[selectedTokenIds[index]] = logits[index];
|
|
2398
|
+
}
|
|
2399
|
+
return {
|
|
2400
|
+
seqLen: this.#state.currentSeqLen,
|
|
2401
|
+
prefixTokens: prefix.tokens,
|
|
2402
|
+
tokens: inputIds,
|
|
2403
|
+
tokenIds: selectedTokenIds,
|
|
2404
|
+
logits,
|
|
2405
|
+
logitsByTokenId,
|
|
2406
|
+
phase: {
|
|
2407
|
+
...(prefillPhase ?? {}),
|
|
2408
|
+
totalMs: performance.now() - totalStart,
|
|
2409
|
+
selectedTokenCount: selectedTokenIds.length,
|
|
2410
|
+
prefixTokens: prefix.tokens.length,
|
|
2411
|
+
},
|
|
2412
|
+
};
|
|
2413
|
+
} finally {
|
|
2414
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2167
2418
|
|
|
2168
2419
|
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
2169
2420
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
@@ -2211,6 +2462,11 @@ export class PipelineGenerator {
|
|
|
2211
2462
|
this.#state.stats.decodeReadbackMapWaitMs = 0;
|
|
2212
2463
|
this.#state.stats.decodeReadbackCleanupMs = 0;
|
|
2213
2464
|
this.#state.stats.decodeReadbackCopyMs = 0;
|
|
2465
|
+
this.#state.stats.prefillRecordMs = 0;
|
|
2466
|
+
this.#state.stats.prefillRecordOps = 0;
|
|
2467
|
+
this.#state.stats.prefillRecordPasses = 0;
|
|
2468
|
+
this.#state.stats.prefillRecordOpLabels = {};
|
|
2469
|
+
this.#state.stats.prefillSubmitWaitMs = 0;
|
|
2214
2470
|
this.#state.stats.ttftMs = 0;
|
|
2215
2471
|
const startTime = performance.now();
|
|
2216
2472
|
|
|
@@ -2411,6 +2667,7 @@ export class PipelineGenerator {
|
|
|
2411
2667
|
maxDecodeTokens: opts.maxTokens,
|
|
2412
2668
|
numLayers: this.#state.modelConfig.numLayers,
|
|
2413
2669
|
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
2670
|
+
configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
|
|
2414
2671
|
});
|
|
2415
2672
|
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
2416
2673
|
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
@@ -2537,7 +2794,10 @@ export class PipelineGenerator {
|
|
|
2537
2794
|
// and speculative use the same weights and state. The benefit is
|
|
2538
2795
|
// amortizing per-iteration overhead for models where batch decode is
|
|
2539
2796
|
// disabled (e.g., linear attention).
|
|
2540
|
-
const speculativeBurstTokens =
|
|
2797
|
+
const speculativeBurstTokens = opts.speculation.tokens;
|
|
2798
|
+
if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
|
|
2799
|
+
throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
|
|
2800
|
+
}
|
|
2541
2801
|
const doSpecDecode = hasLinearLayers
|
|
2542
2802
|
? () => this._decodeStep(generatedIds, opts)
|
|
2543
2803
|
: decodeSingleTokenViaLogits;
|
|
@@ -2721,6 +2981,11 @@ export class PipelineGenerator {
|
|
|
2721
2981
|
async _prefill(inputIds, opts) {
|
|
2722
2982
|
const numTokens = inputIds.length;
|
|
2723
2983
|
const config = this.#state.modelConfig;
|
|
2984
|
+
const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
|
|
2985
|
+
opts?._selectedLogitTokenIds,
|
|
2986
|
+
config.vocabSize,
|
|
2987
|
+
'_prefill.selectedLogitTokenIds'
|
|
2988
|
+
);
|
|
2724
2989
|
const startPos = this.#state.currentSeqLen;
|
|
2725
2990
|
const tracePrefillEnabled = isTraceEnabled('perf');
|
|
2726
2991
|
const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
|
|
@@ -2783,7 +3048,7 @@ export class PipelineGenerator {
|
|
|
2783
3048
|
if (!device || disableCommandBatching) return undefined;
|
|
2784
3049
|
return opts.profile
|
|
2785
3050
|
? createProfilingRecorder(label, device)
|
|
2786
|
-
: createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
|
|
3051
|
+
: createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
|
|
2787
3052
|
};
|
|
2788
3053
|
const recorder = createRecorder('prefill');
|
|
2789
3054
|
const debugCheckBuffer = this.#state.debug
|
|
@@ -2800,6 +3065,9 @@ export class PipelineGenerator {
|
|
|
2800
3065
|
);
|
|
2801
3066
|
context.currentTokenIds = inputIds;
|
|
2802
3067
|
context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
|
|
3068
|
+
context.skipKVCacheWrites = returnHidden
|
|
3069
|
+
&& opts?._embeddingOnly === true
|
|
3070
|
+
&& this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
|
|
2803
3071
|
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
2804
3072
|
? null
|
|
2805
3073
|
: {
|
|
@@ -2863,6 +3131,7 @@ export class PipelineGenerator {
|
|
|
2863
3131
|
hiddenSize: config.hiddenSize,
|
|
2864
3132
|
vocabSize: config.vocabSize,
|
|
2865
3133
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
3134
|
+
embeddingScale: config.embeddingScale,
|
|
2866
3135
|
debug: opts.debug,
|
|
2867
3136
|
recorder,
|
|
2868
3137
|
transpose: this.#state.embeddingTranspose,
|
|
@@ -2933,6 +3202,7 @@ export class PipelineGenerator {
|
|
|
2933
3202
|
hiddenStates.shape,
|
|
2934
3203
|
hiddenStates.label
|
|
2935
3204
|
);
|
|
3205
|
+
recordPrefillRecorderStats(this.#state, recorder);
|
|
2936
3206
|
await recorder.submitAndWait();
|
|
2937
3207
|
await recordProfile(recorder);
|
|
2938
3208
|
}
|
|
@@ -2967,6 +3237,7 @@ export class PipelineGenerator {
|
|
|
2967
3237
|
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2968
3238
|
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2969
3239
|
const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
|
|
3240
|
+
configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
|
|
2970
3241
|
hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
|
|
2971
3242
|
numTokens,
|
|
2972
3243
|
});
|
|
@@ -3023,6 +3294,7 @@ export class PipelineGenerator {
|
|
|
3023
3294
|
currentRecorder,
|
|
3024
3295
|
'prefill_checkpoint_carry'
|
|
3025
3296
|
);
|
|
3297
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3026
3298
|
await currentRecorder.submitAndWait();
|
|
3027
3299
|
await recordProfile(currentRecorder);
|
|
3028
3300
|
currentRecorder = undefined;
|
|
@@ -3035,6 +3307,7 @@ export class PipelineGenerator {
|
|
|
3035
3307
|
'prefill_trace_layer_health_carry'
|
|
3036
3308
|
);
|
|
3037
3309
|
const traceSubmitStart = performance.now();
|
|
3310
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3038
3311
|
await currentRecorder.submitAndWait();
|
|
3039
3312
|
await recordProfile(currentRecorder);
|
|
3040
3313
|
prefillSubmitWaitMs += performance.now() - traceSubmitStart;
|
|
@@ -3108,6 +3381,7 @@ export class PipelineGenerator {
|
|
|
3108
3381
|
this.#state.modelConfig
|
|
3109
3382
|
);
|
|
3110
3383
|
const chunkSubmitStart = performance.now();
|
|
3384
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3111
3385
|
if (chunkSubmitMode === 'async' && !opts.profile) {
|
|
3112
3386
|
currentRecorder.submit({ cleanup: 'deferred' });
|
|
3113
3387
|
} else {
|
|
@@ -3166,6 +3440,7 @@ export class PipelineGenerator {
|
|
|
3166
3440
|
currentRecorder,
|
|
3167
3441
|
'prefill_tsir_drain_carry'
|
|
3168
3442
|
);
|
|
3443
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3169
3444
|
await currentRecorder.submitAndWait();
|
|
3170
3445
|
await recordProfile(currentRecorder);
|
|
3171
3446
|
currentRecorder = undefined;
|
|
@@ -3176,13 +3451,14 @@ export class PipelineGenerator {
|
|
|
3176
3451
|
if (this.#state.finitenessBuffer) {
|
|
3177
3452
|
if (currentRecorder) {
|
|
3178
3453
|
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3454
|
+
currentHiddenBuffer,
|
|
3455
|
+
currentRecorder,
|
|
3456
|
+
'prefill_finiteness_carry'
|
|
3457
|
+
);
|
|
3458
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3459
|
+
await currentRecorder.submitAndWait();
|
|
3460
|
+
await recordProfile(currentRecorder);
|
|
3461
|
+
currentRecorder = undefined;
|
|
3186
3462
|
}
|
|
3187
3463
|
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
3188
3464
|
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
@@ -3246,6 +3522,7 @@ export class PipelineGenerator {
|
|
|
3246
3522
|
let usedRecordedLogits = false;
|
|
3247
3523
|
const lmHead = this.#state.weights.get('lm_head');
|
|
3248
3524
|
const canRecordLogits = !!currentRecorder
|
|
3525
|
+
&& selectedLogitTokenIds === null
|
|
3249
3526
|
&& !!lmHead
|
|
3250
3527
|
&& !isCpuWeightBuffer(lmHead)
|
|
3251
3528
|
&& !this.#state.disableRecordedLogits
|
|
@@ -3263,6 +3540,7 @@ export class PipelineGenerator {
|
|
|
3263
3540
|
logitsVocabSize = recorded.vocabSize;
|
|
3264
3541
|
usedRecordedLogits = true;
|
|
3265
3542
|
|
|
3543
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3266
3544
|
await currentRecorder.submitAndWait();
|
|
3267
3545
|
await recordProfile(currentRecorder);
|
|
3268
3546
|
|
|
@@ -3324,6 +3602,7 @@ export class PipelineGenerator {
|
|
|
3324
3602
|
currentRecorder,
|
|
3325
3603
|
'prefill_logits_carry'
|
|
3326
3604
|
);
|
|
3605
|
+
recordPrefillRecorderStats(this.#state, currentRecorder);
|
|
3327
3606
|
await currentRecorder.submitAndWait();
|
|
3328
3607
|
await recordProfile(currentRecorder);
|
|
3329
3608
|
}
|
|
@@ -3337,13 +3616,17 @@ export class PipelineGenerator {
|
|
|
3337
3616
|
undefined,
|
|
3338
3617
|
debugCheckBuffer,
|
|
3339
3618
|
this.#state.runtimeConfig.shared.debug.probes,
|
|
3340
|
-
{ lastPositionOnly: true },
|
|
3619
|
+
{ lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
|
|
3341
3620
|
this.#state.operatorDiagnostics
|
|
3342
3621
|
);
|
|
3343
3622
|
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3623
|
+
if (selectedLogitTokenIds !== null) {
|
|
3624
|
+
lastLogits = logits;
|
|
3625
|
+
} else {
|
|
3626
|
+
lastLogits = logits.length === logitsVocabSize
|
|
3627
|
+
? logits
|
|
3628
|
+
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
3629
|
+
}
|
|
3347
3630
|
releaseBuffer(currentHiddenBuffer);
|
|
3348
3631
|
if (tracePrefillEnabled) {
|
|
3349
3632
|
trace.perf('Prefill logits complete', {
|
|
@@ -3368,11 +3651,11 @@ export class PipelineGenerator {
|
|
|
3368
3651
|
}
|
|
3369
3652
|
}
|
|
3370
3653
|
|
|
3371
|
-
if (opts.debug) {
|
|
3654
|
+
if (opts.debug && selectedLogitTokenIds === null) {
|
|
3372
3655
|
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
3373
3656
|
}
|
|
3374
3657
|
if (isTraceEnabled('logits')) {
|
|
3375
|
-
trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3658
|
+
trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
|
|
3376
3659
|
}
|
|
3377
3660
|
|
|
3378
3661
|
if (opts.debug) {
|