doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { getDevice, setTrackSubmits } from '../../../gpu/device.js';
|
|
4
|
-
import { releaseBuffer, readBuffer, readBufferSlice, uploadData } from '../../../memory/buffer-pool.js';
|
|
4
|
+
import { acquireBuffer, releaseBuffer, readBuffer, readBufferSlice, uploadData } from '../../../memory/buffer-pool.js';
|
|
5
5
|
import { isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
|
|
6
6
|
import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selection-cache.js';
|
|
7
7
|
import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
|
|
@@ -14,12 +14,14 @@ import {
|
|
|
14
14
|
validateCaptureConfig,
|
|
15
15
|
} from '../../../debug/index.js';
|
|
16
16
|
import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
17
|
+
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
17
18
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
19
|
+
import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
|
|
20
|
+
import { f32ToF16Array } from '../../kv-cache/types.js';
|
|
18
21
|
|
|
19
22
|
// Pipeline sub-modules
|
|
20
23
|
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
21
|
-
import {
|
|
22
|
-
import { applyChatTemplate, isStopToken } from './init.js';
|
|
24
|
+
import { applyChatTemplate, createKVCache, isStopToken } from './init.js';
|
|
23
25
|
import { formatChatMessages } from './chat-format.js';
|
|
24
26
|
import { embed } from './embed.js';
|
|
25
27
|
import { processLayer } from './layer.js';
|
|
@@ -36,7 +38,13 @@ import {
|
|
|
36
38
|
FinitenessError,
|
|
37
39
|
advanceWithTokenAndEmbedding as runAdvanceWithTokenAndEmbedding,
|
|
38
40
|
} from './generator-steps.js';
|
|
39
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
buildLayerContext,
|
|
43
|
+
debugCheckBuffer as debugCheckBufferHelper,
|
|
44
|
+
getLogitsConfig,
|
|
45
|
+
getLogitsWeights,
|
|
46
|
+
releaseSharedAttentionState,
|
|
47
|
+
} from './generator-helpers.js';
|
|
40
48
|
import {
|
|
41
49
|
assertTokenIdsInRange,
|
|
42
50
|
assertTokenIdInRange,
|
|
@@ -55,8 +63,10 @@ import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
|
55
63
|
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
56
64
|
import {
|
|
57
65
|
activateFallbackExecutionPlan,
|
|
66
|
+
hasFallbackExecutionPlan,
|
|
58
67
|
rebaseExecutionSessionPlan,
|
|
59
68
|
resetActiveExecutionPlan,
|
|
69
|
+
resolveMaxBatchDecodeTokens,
|
|
60
70
|
resolveActiveExecutionPlan,
|
|
61
71
|
setActiveExecutionPlan,
|
|
62
72
|
} from './execution-plan.js';
|
|
@@ -66,6 +76,26 @@ import {
|
|
|
66
76
|
resetLinearAttentionRuntime,
|
|
67
77
|
restoreLinearAttentionRuntime,
|
|
68
78
|
} from './linear-attention.js';
|
|
79
|
+
import {
|
|
80
|
+
preparePerLayerInputs,
|
|
81
|
+
createPleBufferCache,
|
|
82
|
+
prefetchPerLayerRow,
|
|
83
|
+
ensurePleScaledProjectionNormWeight,
|
|
84
|
+
ensurePleGpuHotVocabularyRuntime,
|
|
85
|
+
ensurePleGpuSplitTablesRuntime,
|
|
86
|
+
getPleHotVocabularyRuntime,
|
|
87
|
+
hasGpuSplitPerLayerInputEmbeddings,
|
|
88
|
+
hasRangeBackedPerLayerInputEmbeddings,
|
|
89
|
+
} from './per-layer-inputs.js';
|
|
90
|
+
import { createTensor } from '../../../gpu/tensor.js';
|
|
91
|
+
import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
|
|
92
|
+
|
|
93
|
+
let intentBundleModulePromise = null;
|
|
94
|
+
|
|
95
|
+
async function getExperimentalIntentBundleModule() {
|
|
96
|
+
intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
|
|
97
|
+
return intentBundleModulePromise;
|
|
98
|
+
}
|
|
69
99
|
|
|
70
100
|
function isStructuredChatRequest(prompt) {
|
|
71
101
|
return prompt != null
|
|
@@ -74,7 +104,71 @@ function isStructuredChatRequest(prompt) {
|
|
|
74
104
|
&& Array.isArray(prompt.messages);
|
|
75
105
|
}
|
|
76
106
|
|
|
107
|
+
export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
|
|
108
|
+
return hitStop !== true
|
|
109
|
+
&& Number.isInteger(actualCount)
|
|
110
|
+
&& Number.isInteger(requestedCount)
|
|
111
|
+
&& actualCount > 0
|
|
112
|
+
&& actualCount < requestedCount;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function resolveHotVocabularyBatchDecodeAvailability({
|
|
116
|
+
hasRangeBackedPerLayerInputs,
|
|
117
|
+
pleHotVocabularyRuntime,
|
|
118
|
+
tokenId,
|
|
119
|
+
}) {
|
|
120
|
+
if (hasRangeBackedPerLayerInputs !== true) {
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
|
|
127
|
+
if (!Number.isInteger(sentinelIndex)) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
if (!Number.isInteger(tokenId)) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
|
|
134
|
+
return hotTokenIndex !== sentinelIndex;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
138
|
+
await ensurePleGpuHotVocabularyRuntime({
|
|
139
|
+
config: state.modelConfig,
|
|
140
|
+
weights: state.weights,
|
|
141
|
+
perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
|
|
142
|
+
debugFlags: state.debugFlags,
|
|
143
|
+
tokenizer: state.tokenizer ?? null,
|
|
144
|
+
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
145
|
+
});
|
|
146
|
+
await ensurePleGpuSplitTablesRuntime({
|
|
147
|
+
config: state.modelConfig,
|
|
148
|
+
weights: state.weights,
|
|
149
|
+
perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
|
|
150
|
+
debugFlags: state.debugFlags,
|
|
151
|
+
});
|
|
152
|
+
await ensurePleScaledProjectionNormWeight({
|
|
153
|
+
config: state.modelConfig,
|
|
154
|
+
weights: state.weights,
|
|
155
|
+
weightConfig: {
|
|
156
|
+
rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
|
|
157
|
+
},
|
|
158
|
+
debugFlags: state.debugFlags,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function recordPrefillProfileStep(state, entry) {
|
|
163
|
+
if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
|
|
164
|
+
if (!state.stats.prefillProfileSteps) {
|
|
165
|
+
state.stats.prefillProfileSteps = [];
|
|
166
|
+
}
|
|
167
|
+
state.stats.prefillProfileSteps.push(entry);
|
|
168
|
+
}
|
|
169
|
+
|
|
77
170
|
function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
171
|
+
const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
|
|
78
172
|
if (typeof prompt === 'string') {
|
|
79
173
|
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
80
174
|
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
@@ -83,7 +177,7 @@ function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
|
83
177
|
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
84
178
|
);
|
|
85
179
|
}
|
|
86
|
-
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType);
|
|
180
|
+
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
|
|
87
181
|
}
|
|
88
182
|
return prompt;
|
|
89
183
|
}
|
|
@@ -103,7 +197,227 @@ function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
|
103
197
|
);
|
|
104
198
|
}
|
|
105
199
|
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
106
|
-
return formatChatMessages(messages, templateType);
|
|
200
|
+
return formatChatMessages(messages, templateType, chatOptions);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
204
|
+
if (!buffer) {
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
|
|
208
|
+
if (ownsBuffer) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
212
|
+
if (cachedPleBuffer) {
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (recorder) {
|
|
216
|
+
recorder.trackTemporaryBuffer(buffer);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
releaseBuffer(buffer);
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
|
|
223
|
+
if (override == null) {
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
if (typeof override !== 'object') {
|
|
227
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const prefixLength = Number(override.prefixLength);
|
|
231
|
+
if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
|
|
232
|
+
throw new Error(
|
|
233
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
if (prefixLength === 0) {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const offset = Number(override.offset ?? 0);
|
|
241
|
+
if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
|
|
242
|
+
throw new Error(
|
|
243
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
if (offset + prefixLength > numTokens) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
`[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const embeddings = override.embeddings ?? null;
|
|
253
|
+
if (!embeddings) {
|
|
254
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const expectedLength = prefixLength * hiddenSize;
|
|
258
|
+
if (embeddings instanceof Float32Array) {
|
|
259
|
+
if (embeddings.length !== expectedLength) {
|
|
260
|
+
throw new Error(
|
|
261
|
+
`[Pipeline] ${contextLabel}: embedding override length mismatch ` +
|
|
262
|
+
`(expected=${expectedLength}, got=${embeddings.length}).`
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
} else if (isGpuBufferInstance(embeddings)) {
|
|
266
|
+
const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
|
|
267
|
+
if (embeddings.size < expectedBytes) {
|
|
268
|
+
throw new Error(
|
|
269
|
+
`[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
|
|
270
|
+
`(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
} else {
|
|
274
|
+
throw new Error(
|
|
275
|
+
`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return {
|
|
280
|
+
prefixLength,
|
|
281
|
+
offset,
|
|
282
|
+
embeddings,
|
|
283
|
+
expectedLength,
|
|
284
|
+
byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
|
|
285
|
+
byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
|
|
290
|
+
if (embeddingInputSpan == null) {
|
|
291
|
+
return inputIds;
|
|
292
|
+
}
|
|
293
|
+
if (typeof embeddingInputSpan !== 'object') {
|
|
294
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const offset = Number(embeddingInputSpan.offset);
|
|
298
|
+
const length = Number(embeddingInputSpan.length);
|
|
299
|
+
const tokenId = Number(embeddingInputSpan.tokenId);
|
|
300
|
+
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
301
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
|
|
302
|
+
}
|
|
303
|
+
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
|
|
304
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
|
|
305
|
+
}
|
|
306
|
+
if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
|
|
307
|
+
throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
|
|
308
|
+
}
|
|
309
|
+
if (offset + length > inputIds.length) {
|
|
310
|
+
throw new Error(
|
|
311
|
+
`[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
|
|
312
|
+
`exceeds inputIds length ${inputIds.length}.`
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
const replacedInputIds = Array.from(inputIds);
|
|
316
|
+
replacedInputIds.fill(tokenId, offset, offset + length);
|
|
317
|
+
return replacedInputIds;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
|
|
321
|
+
if (bidirectionalSpan == null) {
|
|
322
|
+
return null;
|
|
323
|
+
}
|
|
324
|
+
if (typeof bidirectionalSpan !== 'object') {
|
|
325
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const offset = Number(bidirectionalSpan.offset);
|
|
329
|
+
const length = Number(bidirectionalSpan.length);
|
|
330
|
+
if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
|
|
331
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
|
|
332
|
+
}
|
|
333
|
+
if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
|
|
334
|
+
throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
|
|
335
|
+
}
|
|
336
|
+
if ((offset + length) > inputIds.length) {
|
|
337
|
+
throw new Error(
|
|
338
|
+
`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
|
|
339
|
+
`exceeds inputIds length ${inputIds.length}.`
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
return { offset, length };
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
|
|
346
|
+
if (!override) {
|
|
347
|
+
return baseTensor;
|
|
348
|
+
}
|
|
349
|
+
if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
|
|
350
|
+
throw new Error(
|
|
351
|
+
`[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const device = getDevice();
|
|
356
|
+
if (!device) {
|
|
357
|
+
throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
|
|
361
|
+
const targetElementOffset = override.offset * hiddenSize;
|
|
362
|
+
const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
|
|
363
|
+
try {
|
|
364
|
+
const encoder = device.createCommandEncoder();
|
|
365
|
+
encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
|
|
366
|
+
if (baseTensor.dtype === 'f32') {
|
|
367
|
+
if (isGpuBufferInstance(override.embeddings)) {
|
|
368
|
+
encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
|
|
369
|
+
device.queue.submit([encoder.finish()]);
|
|
370
|
+
} else {
|
|
371
|
+
device.queue.submit([encoder.finish()]);
|
|
372
|
+
uploadData(outputBuffer, override.embeddings, override.byteOffset);
|
|
373
|
+
}
|
|
374
|
+
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
if (isGpuBufferInstance(override.embeddings)) {
|
|
378
|
+
const overrideTensor = createTensor(
|
|
379
|
+
override.embeddings,
|
|
380
|
+
'f32',
|
|
381
|
+
[override.prefixLength, hiddenSize],
|
|
382
|
+
'prefill_embedding_override_f32'
|
|
383
|
+
);
|
|
384
|
+
assertImplicitDtypeTransitionAllowed({
|
|
385
|
+
executionPolicies,
|
|
386
|
+
fromDtype: 'f32',
|
|
387
|
+
toDtype: 'f16',
|
|
388
|
+
op: 'prefill_embedding_override',
|
|
389
|
+
detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
|
|
390
|
+
});
|
|
391
|
+
const castedOverride = await castF32ToF16(overrideTensor);
|
|
392
|
+
try {
|
|
393
|
+
encoder.copyBufferToBuffer(
|
|
394
|
+
castedOverride.buffer,
|
|
395
|
+
0,
|
|
396
|
+
outputBuffer,
|
|
397
|
+
targetByteOffset,
|
|
398
|
+
castedOverride.buffer.size
|
|
399
|
+
);
|
|
400
|
+
device.queue.submit([encoder.finish()]);
|
|
401
|
+
} finally {
|
|
402
|
+
releaseBuffer(castedOverride.buffer);
|
|
403
|
+
}
|
|
404
|
+
} else {
|
|
405
|
+
assertImplicitDtypeTransitionAllowed({
|
|
406
|
+
executionPolicies,
|
|
407
|
+
fromDtype: 'f32',
|
|
408
|
+
toDtype: 'f16',
|
|
409
|
+
op: 'prefill_embedding_override',
|
|
410
|
+
detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
|
|
411
|
+
});
|
|
412
|
+
const packedOverride = f32ToF16Array(override.embeddings);
|
|
413
|
+
device.queue.submit([encoder.finish()]);
|
|
414
|
+
uploadData(outputBuffer, packedOverride, targetByteOffset);
|
|
415
|
+
}
|
|
416
|
+
return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
|
|
417
|
+
} catch (error) {
|
|
418
|
+
releaseBuffer(outputBuffer);
|
|
419
|
+
throw error;
|
|
420
|
+
}
|
|
107
421
|
}
|
|
108
422
|
|
|
109
423
|
/**
|
|
@@ -147,6 +461,44 @@ function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenTe
|
|
|
147
461
|
return fallbackText;
|
|
148
462
|
}
|
|
149
463
|
|
|
464
|
+
function usesReplayPrefillDecode(state) {
|
|
465
|
+
return state?.modelConfig?.decodeStrategy === 'replay_prefill';
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
|
|
469
|
+
if (
|
|
470
|
+
opts?.disableCommandBatching === true
|
|
471
|
+
|| opts?.debug === true
|
|
472
|
+
|| (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
|
|
473
|
+
) {
|
|
474
|
+
return true;
|
|
475
|
+
}
|
|
476
|
+
if (state?.kvCache?.layout === 'bdpa_paged') {
|
|
477
|
+
return true;
|
|
478
|
+
}
|
|
479
|
+
if (multimodalBidirectionalSpan == null) {
|
|
480
|
+
return false;
|
|
481
|
+
}
|
|
482
|
+
if (state?.kvCache?.hasGPUCache?.() !== true) {
|
|
483
|
+
return false;
|
|
484
|
+
}
|
|
485
|
+
// WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
|
|
486
|
+
// multimodal logits on the first generated token. Keep this lane fail-closed
|
|
487
|
+
// until the recorder/cache interaction is numerically matched to the direct path.
|
|
488
|
+
return true;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
function assertIncrementalDecodeSupport(state, operation) {
|
|
492
|
+
if (!usesReplayPrefillDecode(state)) {
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
throw new Error(
|
|
496
|
+
`[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
|
|
497
|
+
'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
|
|
498
|
+
'explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
|
|
150
502
|
function summarizeExecutionPlan(plan) {
|
|
151
503
|
if (!plan) {
|
|
152
504
|
return null;
|
|
@@ -189,6 +541,64 @@ export function shouldRetryWithFinitenessFallback(error) {
|
|
|
189
541
|
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
190
542
|
}
|
|
191
543
|
|
|
544
|
+
function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
|
|
545
|
+
const activePlan = resolveActiveExecutionPlan(state);
|
|
546
|
+
const wrapped = new Error(
|
|
547
|
+
`[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
|
|
548
|
+
`"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
|
|
549
|
+
'Resolve the unstable path with an explicit capability-aware execution override, ' +
|
|
550
|
+
'or opt into alternate-plan recovery with ' +
|
|
551
|
+
'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
|
|
552
|
+
error instanceof Error ? { cause: error } : undefined
|
|
553
|
+
);
|
|
554
|
+
wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
|
|
555
|
+
return wrapped;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function assertResolvedKVDtype(kvDtype, contextLabel) {
|
|
559
|
+
if (kvDtype === 'f16' || kvDtype === 'f32') {
|
|
560
|
+
return kvDtype;
|
|
561
|
+
}
|
|
562
|
+
throw new Error(
|
|
563
|
+
`[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
|
|
564
|
+
`got ${JSON.stringify(kvDtype)}.`
|
|
565
|
+
);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
function resolveTargetPlanKVDtype(plan, contextLabel) {
|
|
569
|
+
return assertResolvedKVDtype(
|
|
570
|
+
plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
|
|
571
|
+
contextLabel
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
|
|
576
|
+
return assertResolvedKVDtype(
|
|
577
|
+
state?.kvCache?.kvDtype
|
|
578
|
+
?? plan?.kernelPath?.kvDtype
|
|
579
|
+
?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
|
|
580
|
+
?? plan?.activationDtype
|
|
581
|
+
?? null,
|
|
582
|
+
contextLabel
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
|
|
587
|
+
const runtimeInference = state?.runtimeConfig?.inference;
|
|
588
|
+
if (!runtimeInference?.session?.kvcache) {
|
|
589
|
+
throw new Error(
|
|
590
|
+
'[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
|
|
591
|
+
);
|
|
592
|
+
}
|
|
593
|
+
return mergeRuntimeValues(runtimeInference, {
|
|
594
|
+
session: {
|
|
595
|
+
kvcache: {
|
|
596
|
+
kvDtype,
|
|
597
|
+
},
|
|
598
|
+
},
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
|
|
192
602
|
export class PipelineGenerator {
|
|
193
603
|
|
|
194
604
|
#state;
|
|
@@ -222,7 +632,62 @@ export class PipelineGenerator {
|
|
|
222
632
|
return this.#finitenessFallbackWindow !== null;
|
|
223
633
|
}
|
|
224
634
|
|
|
225
|
-
|
|
635
|
+
_resetReplayPrefillRuntimeState() {
|
|
636
|
+
this.#state.kvCache?.clear?.();
|
|
637
|
+
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
638
|
+
this.#state.currentSeqLen = 0;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
async _replayPrefillDecodeLogits(currentIds, opts) {
|
|
642
|
+
// Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
|
|
643
|
+
// Without KV cache creation, this bound is not enforced elsewhere.
|
|
644
|
+
const kvConfig = this.#state.runtimeConfig?.inference?.session?.kvcache;
|
|
645
|
+
const replayMaxSeqLen = kvConfig?.maxSeqLen;
|
|
646
|
+
if (Number.isFinite(replayMaxSeqLen) && replayMaxSeqLen > 0 && currentIds.length > replayMaxSeqLen) {
|
|
647
|
+
throw new Error(
|
|
648
|
+
`[Pipeline] Replay-prefill sequence length ${currentIds.length} exceeds ` +
|
|
649
|
+
`runtime.inference.session.kvcache.maxSeqLen (${replayMaxSeqLen}). ` +
|
|
650
|
+
'Increase maxSeqLen in a tier profile or runtime config to allow longer sequences.'
|
|
651
|
+
);
|
|
652
|
+
}
|
|
653
|
+
this.#state.decodeStepCount++;
|
|
654
|
+
this._resetReplayPrefillRuntimeState();
|
|
655
|
+
const logits = await this._prefill(currentIds, opts);
|
|
656
|
+
return {
|
|
657
|
+
logits,
|
|
658
|
+
logitsBuffer: null,
|
|
659
|
+
logitsDtype: null,
|
|
660
|
+
rawVocabSize: this.#state.modelConfig.vocabSize,
|
|
661
|
+
vocabSize: this.#state.modelConfig.vocabSize,
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
_shouldUseFinitenessFallback(error, contextLabel) {
|
|
666
|
+
if (!shouldRetryWithFinitenessFallback(error)) {
|
|
667
|
+
return false;
|
|
668
|
+
}
|
|
669
|
+
if (!hasFallbackExecutionPlan(this.#state)) {
|
|
670
|
+
throw createUnhandledFinitenessPolicyError(this.#state, contextLabel, error);
|
|
671
|
+
}
|
|
672
|
+
return true;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
_recreateKVCacheForExecutionPlan(plan, reasonLabel) {
|
|
676
|
+
const kvDtype = resolveTargetPlanKVDtype(plan, `${reasonLabel}: target plan`);
|
|
677
|
+
const runtimeInference = cloneRuntimeInferenceWithKVDtype(this.#state, kvDtype);
|
|
678
|
+
this.#state.kvCache?.destroy?.();
|
|
679
|
+
this.#state.kvCache = createKVCache(
|
|
680
|
+
this.#state.modelConfig,
|
|
681
|
+
this.#state.useGPU,
|
|
682
|
+
this.#state.debug,
|
|
683
|
+
runtimeInference
|
|
684
|
+
);
|
|
685
|
+
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
686
|
+
this.#state.currentSeqLen = 0;
|
|
687
|
+
return kvDtype;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
_openFinitenessFallbackWindow(opts, reasonLabel, tokenCount, rollbackSeqLen = undefined) {
|
|
226
691
|
const normalizedCount = Number.isFinite(tokenCount)
|
|
227
692
|
? Math.max(1, Math.floor(tokenCount))
|
|
228
693
|
: 1;
|
|
@@ -233,7 +698,7 @@ export class PipelineGenerator {
|
|
|
233
698
|
);
|
|
234
699
|
return;
|
|
235
700
|
}
|
|
236
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel);
|
|
701
|
+
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
237
702
|
this.#finitenessFallbackWindow = {
|
|
238
703
|
original,
|
|
239
704
|
remainingTokens: normalizedCount,
|
|
@@ -264,6 +729,7 @@ export class PipelineGenerator {
|
|
|
264
729
|
}
|
|
265
730
|
|
|
266
731
|
_resetDecodeRuntimeState() {
|
|
732
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
267
733
|
this.#state.stats.decodeMode = null;
|
|
268
734
|
this.#state.stats.batchGuardReason = null;
|
|
269
735
|
this.#state.stats.decodeProfileSteps = [];
|
|
@@ -292,6 +758,7 @@ export class PipelineGenerator {
|
|
|
292
758
|
buildLayerContext(this.#state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
|
|
293
759
|
getLogitsWeights: () => getLogitsWeights(this.#state),
|
|
294
760
|
getLogitsConfig: () => getLogitsConfig(this.#state),
|
|
761
|
+
releaseSharedAttentionState,
|
|
295
762
|
debugCheckBuffer,
|
|
296
763
|
};
|
|
297
764
|
}
|
|
@@ -319,6 +786,14 @@ export class PipelineGenerator {
|
|
|
319
786
|
return inputIds;
|
|
320
787
|
}
|
|
321
788
|
|
|
789
|
+
_resolvePromptOrInputIds(prompt, useChatTemplate, contextLabel, explicitInputIds = null) {
|
|
790
|
+
if (Array.isArray(explicitInputIds)) {
|
|
791
|
+
this._assertTokenIdsInRange(explicitInputIds, `${contextLabel}.inputIds`);
|
|
792
|
+
return explicitInputIds;
|
|
793
|
+
}
|
|
794
|
+
return this._resolvePromptTokenIds(prompt, useChatTemplate, contextLabel);
|
|
795
|
+
}
|
|
796
|
+
|
|
322
797
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
323
798
|
const sampledLogits = Float32Array.from(logits);
|
|
324
799
|
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
@@ -333,7 +808,8 @@ export class PipelineGenerator {
|
|
|
333
808
|
}
|
|
334
809
|
|
|
335
810
|
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
336
|
-
const
|
|
811
|
+
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
812
|
+
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
|
|
337
813
|
if (opts.debug) {
|
|
338
814
|
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
339
815
|
}
|
|
@@ -342,14 +818,16 @@ export class PipelineGenerator {
|
|
|
342
818
|
try {
|
|
343
819
|
logits = await this._prefill(inputIds, opts);
|
|
344
820
|
} catch (error) {
|
|
345
|
-
if (!
|
|
821
|
+
if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
|
|
346
822
|
throw error;
|
|
347
823
|
}
|
|
348
824
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
|
|
349
|
-
logits = await this.
|
|
825
|
+
logits = await this._retryWithPersistentFinitenessFallback(
|
|
350
826
|
opts,
|
|
351
827
|
contextLabel,
|
|
352
|
-
|
|
828
|
+
opts.maxTokens ?? 1,
|
|
829
|
+
() => this._prefill(inputIds, opts),
|
|
830
|
+
prefillStartSeqLen
|
|
353
831
|
);
|
|
354
832
|
}
|
|
355
833
|
|
|
@@ -357,6 +835,9 @@ export class PipelineGenerator {
|
|
|
357
835
|
}
|
|
358
836
|
|
|
359
837
|
async _decodeStepToLogits(currentIds, opts) {
|
|
838
|
+
if (usesReplayPrefillDecode(this.#state)) {
|
|
839
|
+
return this._replayPrefillDecodeLogits(currentIds, opts);
|
|
840
|
+
}
|
|
360
841
|
const debugCheckBuffer = this.#state.debug
|
|
361
842
|
? (buffer, label, numTokens, expectedDim) =>
|
|
362
843
|
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
@@ -369,6 +850,21 @@ export class PipelineGenerator {
|
|
|
369
850
|
return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
|
|
370
851
|
}
|
|
371
852
|
|
|
853
|
+
_matchesStopSequence(generatedIds, stopSequenceStart, stopSequences) {
|
|
854
|
+
if (!Array.isArray(stopSequences) || stopSequences.length === 0) {
|
|
855
|
+
return false;
|
|
856
|
+
}
|
|
857
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
858
|
+
return stopSequences.some((sequence) => fullText.endsWith(sequence));
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
_shouldStopAfterAppendedToken(generatedIds, tokenId, opts, runtime) {
|
|
862
|
+
if (isStopToken(tokenId, runtime.stopTokenIds, runtime.eosToken)) {
|
|
863
|
+
return true;
|
|
864
|
+
}
|
|
865
|
+
return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
|
|
866
|
+
}
|
|
867
|
+
|
|
372
868
|
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
373
869
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
374
870
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
@@ -385,6 +881,12 @@ export class PipelineGenerator {
|
|
|
385
881
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
386
882
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
387
883
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
884
|
+
this.#state.stats.pleHotVocabularyHits = 0;
|
|
885
|
+
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
886
|
+
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
887
|
+
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
888
|
+
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
889
|
+
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
388
890
|
this.#state.stats.decodeMode = null;
|
|
389
891
|
this.#state.stats.batchGuardReason = null;
|
|
390
892
|
this.#state.stats.ttftMs = 0;
|
|
@@ -444,6 +946,7 @@ export class PipelineGenerator {
|
|
|
444
946
|
};
|
|
445
947
|
|
|
446
948
|
try {
|
|
949
|
+
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
447
950
|
const prefillStart = performance.now();
|
|
448
951
|
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
|
|
449
952
|
let prefillLogits = initialPrefillLogits;
|
|
@@ -464,6 +967,7 @@ export class PipelineGenerator {
|
|
|
464
967
|
?? intentBundle?.constraints?.max_drift_threshold;
|
|
465
968
|
|
|
466
969
|
if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
|
|
970
|
+
const { enforceLogitDrift } = await getExperimentalIntentBundleModule();
|
|
467
971
|
const actualTopK = getTopK(
|
|
468
972
|
prefillLogits,
|
|
469
973
|
expectedTopK.length,
|
|
@@ -488,14 +992,16 @@ export class PipelineGenerator {
|
|
|
488
992
|
try {
|
|
489
993
|
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
490
994
|
} catch (error) {
|
|
491
|
-
if (!
|
|
995
|
+
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
492
996
|
throw error;
|
|
493
997
|
}
|
|
494
998
|
log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
|
|
495
|
-
prefillLogits = await this.
|
|
999
|
+
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
496
1000
|
opts,
|
|
497
1001
|
'prefill-sample',
|
|
498
|
-
|
|
1002
|
+
opts.maxTokens,
|
|
1003
|
+
() => this._prefill(inputIds, opts),
|
|
1004
|
+
prefillStartSeqLen
|
|
499
1005
|
);
|
|
500
1006
|
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
501
1007
|
}
|
|
@@ -505,6 +1011,9 @@ export class PipelineGenerator {
|
|
|
505
1011
|
log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
|
|
506
1012
|
}
|
|
507
1013
|
|
|
1014
|
+
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
1015
|
+
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
1016
|
+
const stopSequenceStart = inputIds.length;
|
|
508
1017
|
generatedIds.push(firstToken);
|
|
509
1018
|
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
510
1019
|
|
|
@@ -515,18 +1024,25 @@ export class PipelineGenerator {
|
|
|
515
1024
|
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
516
1025
|
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
517
1026
|
);
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
523
|
-
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
524
|
-
stopSequenceStart: inputIds.length,
|
|
1027
|
+
const decodeRuntime = {
|
|
1028
|
+
stopTokenIds,
|
|
1029
|
+
eosToken,
|
|
1030
|
+
stopSequenceStart,
|
|
525
1031
|
decodeToken,
|
|
526
1032
|
logBatchPath: opts.debug,
|
|
527
1033
|
emitMode: mode,
|
|
528
|
-
}
|
|
529
|
-
|
|
1034
|
+
};
|
|
1035
|
+
|
|
1036
|
+
yield* emitToken(this, firstToken, decodeToken);
|
|
1037
|
+
|
|
1038
|
+
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1039
|
+
this.#state.stats.decodeTimeMs = 0;
|
|
1040
|
+
this.#state.stats.tokensGenerated = 1;
|
|
1041
|
+
this.#state.stats.decodeTokens = 1;
|
|
1042
|
+
} else {
|
|
1043
|
+
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
1044
|
+
}
|
|
1045
|
+
const tokensGenerated = this.#state.stats.decodeTokens ?? 1;
|
|
530
1046
|
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
531
1047
|
|
|
532
1048
|
if (opts.debug) {
|
|
@@ -566,26 +1082,78 @@ export class PipelineGenerator {
|
|
|
566
1082
|
}
|
|
567
1083
|
}
|
|
568
1084
|
|
|
569
|
-
_beginFinitenessFallback(opts, reasonLabel) {
|
|
1085
|
+
_beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen = undefined) {
|
|
570
1086
|
const originalPlan = resolveActiveExecutionPlan(this.#state);
|
|
1087
|
+
const currentKvDtype = resolveCurrentKVCacheDtype(
|
|
1088
|
+
this.#state,
|
|
1089
|
+
originalPlan,
|
|
1090
|
+
`${reasonLabel}: current plan`
|
|
1091
|
+
);
|
|
571
1092
|
const original = {
|
|
572
1093
|
activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
|
|
573
1094
|
seed: opts.seed,
|
|
1095
|
+
restoreKVCachePlan: null,
|
|
574
1096
|
};
|
|
575
1097
|
|
|
576
1098
|
const fallbackPlan = activateFallbackExecutionPlan(this.#state);
|
|
577
1099
|
if (!fallbackPlan) {
|
|
578
|
-
throw new Error(
|
|
1100
|
+
throw new Error(
|
|
1101
|
+
'[Pipeline] Explicit alternate-plan finiteness recovery is unavailable for this model/runtime configuration.'
|
|
1102
|
+
);
|
|
579
1103
|
}
|
|
580
1104
|
log.warn(
|
|
581
1105
|
'Pipeline',
|
|
582
1106
|
`FinitenessGuard fallback (${reasonLabel}): ` +
|
|
583
1107
|
`${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
|
|
584
1108
|
);
|
|
1109
|
+
const fallbackKvDtype = resolveTargetPlanKVDtype(
|
|
1110
|
+
fallbackPlan,
|
|
1111
|
+
`${reasonLabel}: fallback plan`
|
|
1112
|
+
);
|
|
1113
|
+
|
|
1114
|
+
if (Number.isInteger(rollbackSeqLen) && rollbackSeqLen < 0) {
|
|
1115
|
+
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1116
|
+
throw new Error(
|
|
1117
|
+
`[Pipeline] ${reasonLabel}: rollbackSeqLen must be a non-negative integer when provided.`
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
if (fallbackKvDtype !== currentKvDtype) {
|
|
1122
|
+
if (rollbackSeqLen !== 0) {
|
|
1123
|
+
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1124
|
+
throw new Error(
|
|
1125
|
+
`[Pipeline] ${reasonLabel}: finiteness fallback requires rebuilding the KV cache ` +
|
|
1126
|
+
`${currentKvDtype} -> ${fallbackKvDtype}, which is only supported from a fresh prefill (rollbackSeqLen=0).`
|
|
1127
|
+
);
|
|
1128
|
+
}
|
|
1129
|
+
try {
|
|
1130
|
+
this._recreateKVCacheForExecutionPlan(fallbackPlan, reasonLabel);
|
|
1131
|
+
original.restoreKVCachePlan = originalPlan;
|
|
1132
|
+
} catch (error) {
|
|
1133
|
+
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
1134
|
+
try {
|
|
1135
|
+
this._recreateKVCacheForExecutionPlan(originalPlan, `${reasonLabel}: restore primary`);
|
|
1136
|
+
} catch (restoreError) {
|
|
1137
|
+
log.warn(
|
|
1138
|
+
'Pipeline',
|
|
1139
|
+
`Failed to restore primary KV cache after fallback activation error: ${restoreError}`
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
throw error;
|
|
1143
|
+
}
|
|
1144
|
+
} else if (Number.isInteger(rollbackSeqLen)) {
|
|
1145
|
+
this.#state.kvCache?.truncate(rollbackSeqLen);
|
|
1146
|
+
this.#state.currentSeqLen = rollbackSeqLen;
|
|
1147
|
+
if (rollbackSeqLen === 0) {
|
|
1148
|
+
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
1149
|
+
}
|
|
1150
|
+
} else {
|
|
1151
|
+
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
1152
|
+
}
|
|
585
1153
|
|
|
586
1154
|
this.#state.decodeBuffers?.ensureBuffers({
|
|
587
1155
|
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
588
|
-
intermediateSize: this.#state.modelConfig.
|
|
1156
|
+
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
589
1157
|
activationDtype: fallbackPlan.activationDtype,
|
|
590
1158
|
enablePingPong: true,
|
|
591
1159
|
});
|
|
@@ -619,6 +1187,9 @@ export class PipelineGenerator {
|
|
|
619
1187
|
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
620
1188
|
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
621
1189
|
const restoredPlan = resolveActiveExecutionPlan(this.#state);
|
|
1190
|
+
if (original.restoreKVCachePlan) {
|
|
1191
|
+
this._recreateKVCacheForExecutionPlan(restoredPlan, 'restore-primary-plan');
|
|
1192
|
+
}
|
|
622
1193
|
if (this.#state.stats.executionPlan) {
|
|
623
1194
|
this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
|
|
624
1195
|
this.#state.stats.executionPlan.transitions.push({
|
|
@@ -637,18 +1208,17 @@ export class PipelineGenerator {
|
|
|
637
1208
|
const nextActivationDtype = this._getEffectiveActivationDtype();
|
|
638
1209
|
this.#state.decodeBuffers?.ensureBuffers({
|
|
639
1210
|
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
640
|
-
intermediateSize: this.#state.modelConfig.
|
|
1211
|
+
intermediateSize: this.#state.modelConfig.maxIntermediateSize,
|
|
641
1212
|
activationDtype: nextActivationDtype,
|
|
642
1213
|
enablePingPong: true,
|
|
643
1214
|
});
|
|
644
1215
|
}
|
|
645
1216
|
|
|
646
|
-
async _retryWithFinitenessFallback(opts, reasonLabel, retryFn) {
|
|
1217
|
+
async _retryWithFinitenessFallback(opts, reasonLabel, retryFn, rollbackSeqLen = undefined) {
|
|
647
1218
|
if (this._hasFinitenessFallbackWindow()) {
|
|
648
1219
|
return retryFn();
|
|
649
1220
|
}
|
|
650
|
-
this
|
|
651
|
-
const original = this._beginFinitenessFallback(opts, reasonLabel);
|
|
1221
|
+
const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
|
|
652
1222
|
try {
|
|
653
1223
|
return await retryFn();
|
|
654
1224
|
} finally {
|
|
@@ -656,6 +1226,19 @@ export class PipelineGenerator {
|
|
|
656
1226
|
}
|
|
657
1227
|
}
|
|
658
1228
|
|
|
1229
|
+
async _retryWithPersistentFinitenessFallback(opts, reasonLabel, tokenBudget, retryFn, rollbackSeqLen = undefined) {
|
|
1230
|
+
if (this._hasFinitenessFallbackWindow()) {
|
|
1231
|
+
return retryFn();
|
|
1232
|
+
}
|
|
1233
|
+
this._openFinitenessFallbackWindow(opts, reasonLabel, tokenBudget, rollbackSeqLen);
|
|
1234
|
+
try {
|
|
1235
|
+
return await retryFn();
|
|
1236
|
+
} catch (error) {
|
|
1237
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
1238
|
+
throw error;
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
|
|
659
1242
|
async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
|
|
660
1243
|
const windowTokens = this._resolveDeferredRoundingWindowTokens();
|
|
661
1244
|
if (windowTokens <= 1) {
|
|
@@ -666,7 +1249,6 @@ export class PipelineGenerator {
|
|
|
666
1249
|
);
|
|
667
1250
|
}
|
|
668
1251
|
|
|
669
|
-
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
670
1252
|
this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
|
|
671
1253
|
try {
|
|
672
1254
|
return await this._decodeStep(generatedIds, opts);
|
|
@@ -705,6 +1287,12 @@ export class PipelineGenerator {
|
|
|
705
1287
|
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
706
1288
|
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
707
1289
|
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
1290
|
+
this.#state.stats.pleHotVocabularyHits = 0;
|
|
1291
|
+
this.#state.stats.pleHotVocabularyMisses = 0;
|
|
1292
|
+
this.#state.stats.plePreparedTokenCacheHits = 0;
|
|
1293
|
+
this.#state.stats.plePreparedTokenCacheMisses = 0;
|
|
1294
|
+
this.#state.stats.plePreparedTokenCacheEntries = 0;
|
|
1295
|
+
this.#state.stats.plePreparedTokenCacheBytes = 0;
|
|
708
1296
|
this.#state.stats.decodeMode = null;
|
|
709
1297
|
this.#state.stats.batchGuardReason = null;
|
|
710
1298
|
this.#state.stats.ttftMs = 0;
|
|
@@ -718,6 +1306,7 @@ export class PipelineGenerator {
|
|
|
718
1306
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
719
1307
|
|
|
720
1308
|
try {
|
|
1309
|
+
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
721
1310
|
const prefillStart = performance.now();
|
|
722
1311
|
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
723
1312
|
let prefillLogits = initialPrefillLogits;
|
|
@@ -730,58 +1319,45 @@ export class PipelineGenerator {
|
|
|
730
1319
|
try {
|
|
731
1320
|
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
732
1321
|
} catch (error) {
|
|
733
|
-
if (!
|
|
1322
|
+
if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
|
|
734
1323
|
throw error;
|
|
735
1324
|
}
|
|
736
|
-
prefillLogits = await this.
|
|
1325
|
+
prefillLogits = await this._retryWithPersistentFinitenessFallback(
|
|
737
1326
|
opts,
|
|
738
1327
|
'prefill-sample',
|
|
739
|
-
|
|
1328
|
+
opts.maxTokens,
|
|
1329
|
+
() => this._prefill(inputIds, opts),
|
|
1330
|
+
prefillStartSeqLen
|
|
740
1331
|
);
|
|
741
1332
|
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
742
1333
|
}
|
|
743
1334
|
|
|
744
|
-
generatedIds.push(firstToken);
|
|
745
|
-
const tokenIds = [firstToken];
|
|
746
|
-
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
747
|
-
|
|
748
1335
|
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
749
1336
|
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
750
1337
|
const stopSequenceStart = inputIds.length;
|
|
1338
|
+
generatedIds.push(firstToken);
|
|
1339
|
+
const tokenIds = [firstToken];
|
|
1340
|
+
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
751
1341
|
markKernelCacheWarmed();
|
|
752
|
-
const decodeStart = performance.now();
|
|
753
1342
|
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
);
|
|
766
|
-
} else {
|
|
767
|
-
throw error;
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
generatedIds.push(nextToken);
|
|
771
|
-
tokenIds.push(nextToken);
|
|
772
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
773
|
-
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
774
|
-
break;
|
|
775
|
-
}
|
|
776
|
-
if (opts.stopSequences.length > 0) {
|
|
777
|
-
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
778
|
-
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1343
|
+
const decodeRuntime = {
|
|
1344
|
+
stopTokenIds,
|
|
1345
|
+
eosToken,
|
|
1346
|
+
stopSequenceStart,
|
|
1347
|
+
decodeToken: () => '',
|
|
1348
|
+
emitMode: 'token',
|
|
1349
|
+
};
|
|
1350
|
+
|
|
1351
|
+
if (!this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1352
|
+
for await (const tokenId of this._runDecodeLoop(generatedIds, opts, options, decodeRuntime)) {
|
|
1353
|
+
tokenIds.push(tokenId);
|
|
779
1354
|
}
|
|
1355
|
+
} else {
|
|
1356
|
+
this.#state.stats.decodeTimeMs = 0;
|
|
1357
|
+
this.#state.stats.tokensGenerated = 1;
|
|
1358
|
+
this.#state.stats.decodeTokens = 1;
|
|
780
1359
|
}
|
|
781
1360
|
|
|
782
|
-
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
783
|
-
this.#state.stats.tokensGenerated = tokenIds.length;
|
|
784
|
-
this.#state.stats.decodeTokens = tokenIds.length;
|
|
785
1361
|
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
786
1362
|
|
|
787
1363
|
return {
|
|
@@ -804,63 +1380,72 @@ export class PipelineGenerator {
|
|
|
804
1380
|
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
805
1381
|
throw new Error('Generation already in progress');
|
|
806
1382
|
}
|
|
1383
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillKVOnly');
|
|
807
1384
|
this._resetDecodeRuntimeState();
|
|
808
1385
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1386
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
809
1387
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
810
|
-
const
|
|
1388
|
+
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
1389
|
+
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillKVOnly', opts.inputIds);
|
|
811
1390
|
if (opts.debug) {
|
|
812
1391
|
log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
|
|
813
1392
|
}
|
|
814
1393
|
|
|
815
|
-
let prefillResult;
|
|
816
1394
|
try {
|
|
817
|
-
prefillResult
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
1395
|
+
let prefillResult;
|
|
1396
|
+
try {
|
|
1397
|
+
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
1398
|
+
} catch (error) {
|
|
1399
|
+
if (this._shouldUseFinitenessFallback(error, 'prefillKVOnly')) {
|
|
1400
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
|
|
1401
|
+
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
1402
|
+
opts,
|
|
1403
|
+
'prefillKVOnly',
|
|
1404
|
+
1,
|
|
1405
|
+
() => this._prefillToHidden(inputIds, opts),
|
|
1406
|
+
prefillStartSeqLen
|
|
1407
|
+
);
|
|
1408
|
+
} else {
|
|
1409
|
+
throw error;
|
|
1410
|
+
}
|
|
828
1411
|
}
|
|
829
|
-
}
|
|
830
1412
|
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1413
|
+
const {
|
|
1414
|
+
numTokens,
|
|
1415
|
+
startPos,
|
|
1416
|
+
currentRecorder,
|
|
1417
|
+
recordProfile,
|
|
1418
|
+
currentHiddenBuffer,
|
|
1419
|
+
} = prefillResult;
|
|
1420
|
+
|
|
1421
|
+
// Ensure prefill work completes before returning a usable snapshot.
|
|
1422
|
+
if (currentRecorder) {
|
|
1423
|
+
await currentRecorder.submitAndWait();
|
|
1424
|
+
await recordProfile(currentRecorder);
|
|
1425
|
+
} else {
|
|
1426
|
+
const device = getDevice();
|
|
1427
|
+
if (device) {
|
|
1428
|
+
await device.queue.onSubmittedWorkDone();
|
|
1429
|
+
}
|
|
847
1430
|
}
|
|
848
|
-
}
|
|
849
1431
|
|
|
850
|
-
|
|
851
|
-
|
|
1432
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
1433
|
+
releaseBuffer(currentHiddenBuffer);
|
|
852
1434
|
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1435
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
1436
|
+
if (!snapshot) {
|
|
1437
|
+
throw new Error('KV cache unavailable after prefill');
|
|
1438
|
+
}
|
|
857
1439
|
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
1440
|
+
return {
|
|
1441
|
+
cache: snapshot,
|
|
1442
|
+
seqLen: this.#state.currentSeqLen,
|
|
1443
|
+
tokens: inputIds,
|
|
1444
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
1445
|
+
};
|
|
1446
|
+
} finally {
|
|
1447
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
1448
|
+
}
|
|
864
1449
|
}
|
|
865
1450
|
|
|
866
1451
|
async prefillWithEmbedding(prompt, options = {}) {
|
|
@@ -868,93 +1453,115 @@ export class PipelineGenerator {
|
|
|
868
1453
|
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
869
1454
|
throw new Error('Generation already in progress');
|
|
870
1455
|
}
|
|
1456
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
|
|
871
1457
|
this._resetDecodeRuntimeState();
|
|
872
1458
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1459
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
873
1460
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
874
|
-
const
|
|
1461
|
+
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
1462
|
+
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
875
1463
|
if (opts.debug) {
|
|
876
1464
|
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
877
1465
|
}
|
|
878
1466
|
|
|
879
|
-
let prefillResult;
|
|
880
1467
|
try {
|
|
881
|
-
prefillResult
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
1468
|
+
let prefillResult;
|
|
1469
|
+
try {
|
|
1470
|
+
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
1471
|
+
} catch (error) {
|
|
1472
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
1473
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
1474
|
+
prefillResult = await this._retryWithPersistentFinitenessFallback(
|
|
1475
|
+
opts,
|
|
1476
|
+
'prefillWithEmbedding',
|
|
1477
|
+
1,
|
|
1478
|
+
() => this._prefillToHidden(inputIds, opts),
|
|
1479
|
+
prefillStartSeqLen
|
|
1480
|
+
);
|
|
1481
|
+
} else {
|
|
1482
|
+
throw error;
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
const {
|
|
1487
|
+
numTokens,
|
|
1488
|
+
config,
|
|
1489
|
+
startPos,
|
|
1490
|
+
activationDtype,
|
|
1491
|
+
activationBytes,
|
|
1492
|
+
currentRecorder,
|
|
1493
|
+
recordProfile,
|
|
1494
|
+
currentHiddenBuffer,
|
|
1495
|
+
} = prefillResult;
|
|
1496
|
+
|
|
1497
|
+
// Ensure prefill work completes before readback.
|
|
1498
|
+
if (currentRecorder) {
|
|
1499
|
+
await currentRecorder.submitAndWait();
|
|
1500
|
+
await recordProfile(currentRecorder);
|
|
890
1501
|
} else {
|
|
891
|
-
|
|
1502
|
+
const device = getDevice();
|
|
1503
|
+
if (device) {
|
|
1504
|
+
await device.queue.onSubmittedWorkDone();
|
|
1505
|
+
}
|
|
892
1506
|
}
|
|
893
|
-
}
|
|
894
1507
|
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
config,
|
|
898
|
-
startPos,
|
|
899
|
-
activationDtype,
|
|
900
|
-
activationBytes,
|
|
901
|
-
currentRecorder,
|
|
902
|
-
recordProfile,
|
|
903
|
-
currentHiddenBuffer,
|
|
904
|
-
} = prefillResult;
|
|
905
|
-
|
|
906
|
-
// Ensure prefill work completes before readback.
|
|
907
|
-
if (currentRecorder) {
|
|
908
|
-
await currentRecorder.submitAndWait();
|
|
909
|
-
await recordProfile(currentRecorder);
|
|
910
|
-
} else {
|
|
911
|
-
const device = getDevice();
|
|
912
|
-
if (device) {
|
|
913
|
-
await device.queue.onSubmittedWorkDone();
|
|
1508
|
+
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
1509
|
+
throw new Error('GPU readback disabled; cannot return embedding');
|
|
914
1510
|
}
|
|
915
|
-
}
|
|
916
1511
|
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
1512
|
+
let embedding;
|
|
1513
|
+
try {
|
|
1514
|
+
const hiddenSize = config.hiddenSize;
|
|
1515
|
+
const hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
1516
|
+
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
1517
|
+
if (hiddenData.byteLength === 0) {
|
|
1518
|
+
throw new Error('GPU readback disabled; cannot return embedding');
|
|
1519
|
+
}
|
|
1520
|
+
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
1521
|
+
const finalNormWeights = await this._getFinalNormWeights();
|
|
1522
|
+
embedding = this._extractEmbeddingFromHidden(
|
|
1523
|
+
hiddenStates,
|
|
1524
|
+
numTokens,
|
|
1525
|
+
hiddenSize,
|
|
1526
|
+
opts.embeddingMode,
|
|
1527
|
+
finalNormWeights,
|
|
1528
|
+
config
|
|
1529
|
+
);
|
|
1530
|
+
} finally {
|
|
1531
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1532
|
+
}
|
|
920
1533
|
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1534
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
1535
|
+
|
|
1536
|
+
// Batch embedding skips expensive KV cache clone and linear attention clone
|
|
1537
|
+
// since the caller will reset immediately after extracting the embedding.
|
|
1538
|
+
if (options.__skipStateSnapshot) {
|
|
1539
|
+
return {
|
|
1540
|
+
cache: null,
|
|
1541
|
+
seqLen: this.#state.currentSeqLen,
|
|
1542
|
+
tokens: inputIds,
|
|
1543
|
+
embedding,
|
|
1544
|
+
embeddingMode: opts.embeddingMode,
|
|
1545
|
+
linearAttention: null,
|
|
1546
|
+
};
|
|
928
1547
|
}
|
|
929
|
-
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
930
|
-
const finalNormWeights = await this._getFinalNormWeights();
|
|
931
|
-
embedding = this._extractEmbeddingFromHidden(
|
|
932
|
-
hiddenStates,
|
|
933
|
-
numTokens,
|
|
934
|
-
hiddenSize,
|
|
935
|
-
opts.embeddingMode,
|
|
936
|
-
finalNormWeights,
|
|
937
|
-
config
|
|
938
|
-
);
|
|
939
|
-
} finally {
|
|
940
|
-
releaseBuffer(currentHiddenBuffer);
|
|
941
|
-
}
|
|
942
1548
|
|
|
943
|
-
|
|
1549
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
1550
|
+
if (!snapshot) {
|
|
1551
|
+
throw new Error('KV cache unavailable after prefill');
|
|
1552
|
+
}
|
|
944
1553
|
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
1554
|
+
return {
|
|
1555
|
+
cache: snapshot,
|
|
1556
|
+
seqLen: this.#state.currentSeqLen,
|
|
1557
|
+
tokens: inputIds,
|
|
1558
|
+
embedding,
|
|
1559
|
+
embeddingMode: opts.embeddingMode,
|
|
1560
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
1561
|
+
};
|
|
1562
|
+
} finally {
|
|
1563
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
948
1564
|
}
|
|
949
|
-
|
|
950
|
-
return {
|
|
951
|
-
cache: snapshot,
|
|
952
|
-
seqLen: this.#state.currentSeqLen,
|
|
953
|
-
tokens: inputIds,
|
|
954
|
-
embedding,
|
|
955
|
-
embeddingMode: opts.embeddingMode,
|
|
956
|
-
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
957
|
-
};
|
|
958
1565
|
}
|
|
959
1566
|
|
|
960
1567
|
async prefillWithLogits(prompt, options = {}) {
|
|
@@ -962,29 +1569,36 @@ export class PipelineGenerator {
|
|
|
962
1569
|
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
963
1570
|
throw new Error('Generation already in progress');
|
|
964
1571
|
}
|
|
1572
|
+
assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
|
|
965
1573
|
this._resetDecodeRuntimeState();
|
|
966
1574
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1575
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
967
1576
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
968
|
-
|
|
1577
|
+
try {
|
|
1578
|
+
const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
969
1579
|
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
1580
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
1581
|
+
if (!snapshot) {
|
|
1582
|
+
throw new Error('KV cache unavailable after prefill');
|
|
1583
|
+
}
|
|
974
1584
|
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
1585
|
+
return {
|
|
1586
|
+
cache: snapshot,
|
|
1587
|
+
seqLen: this.#state.currentSeqLen,
|
|
1588
|
+
tokens: inputIds,
|
|
1589
|
+
logits,
|
|
1590
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
1591
|
+
};
|
|
1592
|
+
} finally {
|
|
1593
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
1594
|
+
}
|
|
982
1595
|
}
|
|
983
1596
|
|
|
984
1597
|
|
|
985
1598
|
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
986
1599
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
987
1600
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1601
|
+
assertIncrementalDecodeSupport(this.#state, 'generateWithPrefixKV');
|
|
988
1602
|
|
|
989
1603
|
validateCallTimeOptions(options);
|
|
990
1604
|
|
|
@@ -1016,6 +1630,7 @@ export class PipelineGenerator {
|
|
|
1016
1630
|
resetActiveExecutionPlan(this.#state);
|
|
1017
1631
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1018
1632
|
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
1633
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
1019
1634
|
this.#state.decodeRing?.reset();
|
|
1020
1635
|
this.#state.stats.decodeRecordMs = 0;
|
|
1021
1636
|
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
@@ -1048,9 +1663,15 @@ export class PipelineGenerator {
|
|
|
1048
1663
|
seed: opts.seed,
|
|
1049
1664
|
});
|
|
1050
1665
|
|
|
1666
|
+
const decodeRuntime = {
|
|
1667
|
+
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
1668
|
+
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
1669
|
+
stopSequenceStart: promptTokenCount,
|
|
1670
|
+
decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
|
|
1671
|
+
logBatchPath: false,
|
|
1672
|
+
};
|
|
1051
1673
|
generatedIds.push(firstToken);
|
|
1052
1674
|
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1053
|
-
|
|
1054
1675
|
const firstText = resolveTokenText(
|
|
1055
1676
|
this.#state.tokenizer,
|
|
1056
1677
|
[firstToken],
|
|
@@ -1061,13 +1682,13 @@ export class PipelineGenerator {
|
|
|
1061
1682
|
yield firstText;
|
|
1062
1683
|
if (options.onToken) options.onToken(firstToken, firstText);
|
|
1063
1684
|
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
}
|
|
1685
|
+
if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
|
|
1686
|
+
this.#state.stats.decodeTimeMs = 0;
|
|
1687
|
+
this.#state.stats.tokensGenerated = 1;
|
|
1688
|
+
this.#state.stats.decodeTokens = 1;
|
|
1689
|
+
} else {
|
|
1690
|
+
yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
|
|
1691
|
+
}
|
|
1071
1692
|
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
1072
1693
|
} finally {
|
|
1073
1694
|
this._closeFinitenessFallbackWindow(opts);
|
|
@@ -1093,6 +1714,20 @@ export class PipelineGenerator {
|
|
|
1093
1714
|
let tokensGenerated = 1;
|
|
1094
1715
|
markKernelCacheWarmed();
|
|
1095
1716
|
|
|
1717
|
+
// Step 4: Lazily initialise PLE buffer cache for decode-path slice reuse.
|
|
1718
|
+
const pleHiddenSize = Number(this.#state.modelConfig.hiddenSizePerLayerInput ?? 0);
|
|
1719
|
+
if (pleHiddenSize > 0 && !this.#state.pleCache) {
|
|
1720
|
+
const activationDtype = resolveActiveExecutionPlan(this.#state).activationDtype;
|
|
1721
|
+
const bpe = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
1722
|
+
this.#state.pleCache = createPleBufferCache(
|
|
1723
|
+
this.#state.modelConfig.numLayers,
|
|
1724
|
+
pleHiddenSize * bpe,
|
|
1725
|
+
);
|
|
1726
|
+
}
|
|
1727
|
+
if (pleHiddenSize > 0) {
|
|
1728
|
+
await primePleDecodeRuntimeCache(this.#state, generatedIds);
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1096
1731
|
const decodeStart = performance.now();
|
|
1097
1732
|
const lmHead = this.#state.weights.get('lm_head');
|
|
1098
1733
|
const embedBuffer = this.#state.weights.get('embed');
|
|
@@ -1101,20 +1736,42 @@ export class PipelineGenerator {
|
|
|
1101
1736
|
|| lmHead instanceof Float32Array
|
|
1102
1737
|
|| embedBuffer instanceof Float32Array;
|
|
1103
1738
|
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
1739
|
+
const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
|
|
1104
1740
|
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
1105
|
-
const
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1741
|
+
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
1742
|
+
config: this.#state.modelConfig,
|
|
1743
|
+
weights: this.#state.weights,
|
|
1744
|
+
});
|
|
1745
|
+
const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
|
|
1746
|
+
config: this.#state.modelConfig,
|
|
1747
|
+
weights: this.#state.weights,
|
|
1748
|
+
});
|
|
1749
|
+
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
|
|
1750
|
+
const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
|
|
1751
|
+
hasRangeBackedPerLayerInputs,
|
|
1752
|
+
pleHotVocabularyRuntime,
|
|
1753
|
+
tokenId: generatedIds[generatedIds.length - 1] ?? null,
|
|
1114
1754
|
});
|
|
1755
|
+
const initialHotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
1756
|
+
const executionPlan = opts.executionPlan;
|
|
1757
|
+
let useBatchPath = replayPrefillDecode
|
|
1758
|
+
? false
|
|
1759
|
+
: shouldUseBatchDecode({
|
|
1760
|
+
batchSize: executionPlan.batchSize,
|
|
1761
|
+
useGPU: this.#state.useGPU,
|
|
1762
|
+
gpuSamplingAvailable,
|
|
1763
|
+
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
|
|
1764
|
+
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
1765
|
+
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
1766
|
+
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
1767
|
+
hasLinearAttentionLayers: hasLinearLayers,
|
|
1768
|
+
selfSpeculationEnabled: opts.speculation?.mode === 'self' && !initialHotVocabularyBatchDecodeAvailable,
|
|
1769
|
+
hasRangeBackedPerLayerInputs,
|
|
1770
|
+
});
|
|
1115
1771
|
if (!useBatchPath) {
|
|
1116
1772
|
let reason = null;
|
|
1117
|
-
if (
|
|
1773
|
+
if (replayPrefillDecode) reason = 'replay_prefill_decode';
|
|
1774
|
+
else if (hasCpuWeights) reason = 'cpu_weights';
|
|
1118
1775
|
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
1119
1776
|
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
1120
1777
|
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
@@ -1122,10 +1779,15 @@ export class PipelineGenerator {
|
|
|
1122
1779
|
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
1123
1780
|
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
1124
1781
|
else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
|
|
1125
|
-
|
|
1782
|
+
else if (hasLinearLayers) reason = 'linear_attention_layers';
|
|
1783
|
+
this.#state.stats.decodeMode = replayPrefillDecode
|
|
1784
|
+
? 'replay_prefill'
|
|
1785
|
+
: (opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token');
|
|
1126
1786
|
this.#state.stats.batchGuardReason = reason;
|
|
1127
1787
|
} else {
|
|
1128
|
-
this.#state.stats.decodeMode =
|
|
1788
|
+
this.#state.stats.decodeMode = hasRangeBackedPerLayerInputs
|
|
1789
|
+
? 'batched_gpu_stepwise_ple'
|
|
1790
|
+
: 'batched_gpu';
|
|
1129
1791
|
this.#state.stats.batchGuardReason = null;
|
|
1130
1792
|
}
|
|
1131
1793
|
|
|
@@ -1147,10 +1809,20 @@ export class PipelineGenerator {
|
|
|
1147
1809
|
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
1148
1810
|
useBatchPath = false;
|
|
1149
1811
|
}
|
|
1812
|
+
const hotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
|
|
1150
1813
|
|
|
1151
1814
|
if (useBatchPath) {
|
|
1152
1815
|
const remaining = opts.maxTokens - tokensGenerated;
|
|
1153
|
-
const
|
|
1816
|
+
const maxBatchDecodeTokens = resolveMaxBatchDecodeTokens({
|
|
1817
|
+
hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
|
|
1818
|
+
hasGpuSplitPerLayerInputs,
|
|
1819
|
+
hasLinearAttentionLayers: hasLinearLayers,
|
|
1820
|
+
});
|
|
1821
|
+
const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
|
|
1822
|
+
const boundedBatchTokens = maxBatchDecodeTokens == null
|
|
1823
|
+
? requestedBatchTokens
|
|
1824
|
+
: Math.min(requestedBatchTokens, maxBatchDecodeTokens);
|
|
1825
|
+
const thisBatchSize = Math.min(boundedBatchTokens, remaining);
|
|
1154
1826
|
const lastToken = generatedIds[generatedIds.length - 1];
|
|
1155
1827
|
|
|
1156
1828
|
try {
|
|
@@ -1180,11 +1852,19 @@ export class PipelineGenerator {
|
|
|
1180
1852
|
}
|
|
1181
1853
|
}
|
|
1182
1854
|
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
1183
|
-
if (hitStop || batchResult.actualCount < thisBatchSize) break;
|
|
1184
1855
|
if (opts.stopSequences.length > 0) {
|
|
1185
1856
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1186
1857
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1187
1858
|
}
|
|
1859
|
+
if (hitStop) break;
|
|
1860
|
+
if (shouldDisableBatchDecodeAfterShortBatch({
|
|
1861
|
+
hitStop,
|
|
1862
|
+
actualCount: batchResult.actualCount,
|
|
1863
|
+
requestedCount: thisBatchSize,
|
|
1864
|
+
})) {
|
|
1865
|
+
useBatchPath = false;
|
|
1866
|
+
continue;
|
|
1867
|
+
}
|
|
1188
1868
|
} catch (error) {
|
|
1189
1869
|
log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
|
|
1190
1870
|
useBatchPath = false;
|
|
@@ -1192,7 +1872,7 @@ export class PipelineGenerator {
|
|
|
1192
1872
|
try {
|
|
1193
1873
|
nextToken = await decodeSingleTokenViaLogits();
|
|
1194
1874
|
} catch (singleTokenError) {
|
|
1195
|
-
if (
|
|
1875
|
+
if (this._shouldUseFinitenessFallback(singleTokenError, `decode-batch-step-${tokensGenerated}`)) {
|
|
1196
1876
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
1197
1877
|
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
1198
1878
|
generatedIds,
|
|
@@ -1217,11 +1897,13 @@ export class PipelineGenerator {
|
|
|
1217
1897
|
if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
|
|
1218
1898
|
}
|
|
1219
1899
|
} else if (opts.speculation?.mode === 'self') {
|
|
1220
|
-
// Self-speculation: decode
|
|
1221
|
-
// Same-model speculation always
|
|
1222
|
-
//
|
|
1223
|
-
// state. The benefit is
|
|
1224
|
-
// where batch decode is
|
|
1900
|
+
// Self-speculation: decode one base token plus a configurable burst of
|
|
1901
|
+
// speculative tokens per iteration. Same-model speculation always
|
|
1902
|
+
// accepts under greedy because the model is deterministic — both base
|
|
1903
|
+
// and speculative use the same weights and state. The benefit is
|
|
1904
|
+
// amortizing per-iteration overhead for models where batch decode is
|
|
1905
|
+
// disabled (e.g., linear attention).
|
|
1906
|
+
const speculativeBurstTokens = Math.max(1, Math.trunc(opts.speculation?.tokens ?? 1));
|
|
1225
1907
|
const doSpecDecode = hasLinearLayers
|
|
1226
1908
|
? () => this._decodeStep(generatedIds, opts)
|
|
1227
1909
|
: decodeSingleTokenViaLogits;
|
|
@@ -1231,7 +1913,7 @@ export class PipelineGenerator {
|
|
|
1231
1913
|
try {
|
|
1232
1914
|
baseToken = await doSpecDecode();
|
|
1233
1915
|
} catch (error) {
|
|
1234
|
-
if (
|
|
1916
|
+
if (this._shouldUseFinitenessFallback(error, `spec-base-${tokensGenerated}`)) {
|
|
1235
1917
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
|
|
1236
1918
|
baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
|
|
1237
1919
|
} else {
|
|
@@ -1257,39 +1939,52 @@ export class PipelineGenerator {
|
|
|
1257
1939
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1258
1940
|
}
|
|
1259
1941
|
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1942
|
+
for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
|
|
1943
|
+
if (tokensGenerated >= opts.maxTokens) {
|
|
1944
|
+
break;
|
|
1945
|
+
}
|
|
1946
|
+
let specToken;
|
|
1947
|
+
try {
|
|
1948
|
+
specToken = await doSpecDecode();
|
|
1949
|
+
} catch (error) {
|
|
1950
|
+
if (this._shouldUseFinitenessFallback(error, `spec-extra-${tokensGenerated}`)) {
|
|
1951
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
|
|
1952
|
+
specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
|
|
1953
|
+
} else {
|
|
1954
|
+
throw error;
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
generatedIds.push(specToken);
|
|
1958
|
+
tokensGenerated++;
|
|
1959
|
+
this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
|
|
1960
|
+
this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
|
|
1961
|
+
if (emitMode === 'token') {
|
|
1962
|
+
yield specToken;
|
|
1963
|
+
if (options.onToken) options.onToken(specToken, '');
|
|
1268
1964
|
} else {
|
|
1269
|
-
|
|
1965
|
+
const text = decodeToken(specToken);
|
|
1966
|
+
yield text;
|
|
1967
|
+
if (options.onToken) options.onToken(specToken, text);
|
|
1270
1968
|
}
|
|
1271
|
-
|
|
1272
|
-
generatedIds.push(specToken);
|
|
1273
|
-
tokensGenerated++;
|
|
1274
|
-
this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
|
|
1275
|
-
this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
|
|
1276
|
-
if (emitMode === 'token') {
|
|
1277
|
-
yield specToken;
|
|
1278
|
-
if (options.onToken) options.onToken(specToken, '');
|
|
1279
|
-
} else {
|
|
1280
|
-
const text = decodeToken(specToken);
|
|
1281
|
-
yield text;
|
|
1282
|
-
if (options.onToken) options.onToken(specToken, text);
|
|
1283
|
-
}
|
|
1284
|
-
this._consumeFinitenessFallbackToken(opts);
|
|
1969
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
1285
1970
|
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1971
|
+
if (opts.debug || opts.benchmark) {
|
|
1972
|
+
const elapsedMs = performance.now() - decodeStart;
|
|
1973
|
+
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
1974
|
+
log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
1975
|
+
}
|
|
1291
1976
|
|
|
1292
|
-
|
|
1977
|
+
if (isStopToken(specToken, stopTokenIds, eosToken)) {
|
|
1978
|
+
break;
|
|
1979
|
+
}
|
|
1980
|
+
if (opts.stopSequences.length > 0) {
|
|
1981
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1982
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
|
|
1983
|
+
break;
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) break;
|
|
1293
1988
|
if (opts.stopSequences.length > 0) {
|
|
1294
1989
|
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1295
1990
|
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
@@ -1302,7 +1997,7 @@ export class PipelineGenerator {
|
|
|
1302
1997
|
? await this._decodeStep(generatedIds, opts)
|
|
1303
1998
|
: await decodeSingleTokenViaLogits();
|
|
1304
1999
|
} catch (error) {
|
|
1305
|
-
if (
|
|
2000
|
+
if (this._shouldUseFinitenessFallback(error, `decode-step-${tokensGenerated}`)) {
|
|
1306
2001
|
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
1307
2002
|
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
1308
2003
|
generatedIds,
|
|
@@ -1316,6 +2011,20 @@ export class PipelineGenerator {
|
|
|
1316
2011
|
const tokenTime = performance.now() - tokenStart;
|
|
1317
2012
|
generatedIds.push(nextToken);
|
|
1318
2013
|
tokensGenerated++;
|
|
2014
|
+
|
|
2015
|
+
// Step 5: Fire-and-forget prefetch of next token's PLE row.
|
|
2016
|
+
if (pleHiddenSize > 0) {
|
|
2017
|
+
const pleWeights = this.#state.weights.get('per_layer_inputs');
|
|
2018
|
+
if (pleWeights?.embedTokensPerLayer) {
|
|
2019
|
+
this.#state.plePrefetchPending = prefetchPerLayerRow(
|
|
2020
|
+
nextToken,
|
|
2021
|
+
pleWeights.embedTokensPerLayer,
|
|
2022
|
+
this.#state.modelConfig.numLayers * pleHiddenSize,
|
|
2023
|
+
this.#state.runtimeConfig.inference.session?.perLayerInputs ?? this.#state.modelConfig.perLayerInputsSession,
|
|
2024
|
+
);
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
|
|
1319
2028
|
const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
|
|
1320
2029
|
if (emitMode === 'token') {
|
|
1321
2030
|
yield nextToken;
|
|
@@ -1356,7 +2065,27 @@ export class PipelineGenerator {
|
|
|
1356
2065
|
const config = this.#state.modelConfig;
|
|
1357
2066
|
const startPos = this.#state.currentSeqLen;
|
|
1358
2067
|
const returnHidden = opts?._returnHidden === true;
|
|
2068
|
+
const embeddingInputIds = resolvePrefillEmbeddingInputIds(
|
|
2069
|
+
inputIds,
|
|
2070
|
+
opts?.embeddingInputSpan ?? null,
|
|
2071
|
+
'_prefill'
|
|
2072
|
+
);
|
|
2073
|
+
const multimodalBidirectionalSpan = resolvePrefillMultimodalBidirectionalSpan(
|
|
2074
|
+
inputIds,
|
|
2075
|
+
opts?.multimodalBidirectionalSpan ?? null,
|
|
2076
|
+
'_prefill'
|
|
2077
|
+
);
|
|
2078
|
+
if (embeddingInputIds !== inputIds) {
|
|
2079
|
+
this._assertTokenIdsInRange(embeddingInputIds, '_prefill.embeddingInputIds');
|
|
2080
|
+
}
|
|
2081
|
+
const embeddingOverride = normalizePrefixEmbeddingOverride(
|
|
2082
|
+
opts?.embeddingOverrides ?? null,
|
|
2083
|
+
config.hiddenSize,
|
|
2084
|
+
numTokens,
|
|
2085
|
+
'_prefill'
|
|
2086
|
+
);
|
|
1359
2087
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2088
|
+
this.#state.stats.prefillProfileSteps = [];
|
|
1360
2089
|
|
|
1361
2090
|
if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
|
|
1362
2091
|
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
@@ -1384,9 +2113,11 @@ export class PipelineGenerator {
|
|
|
1384
2113
|
|
|
1385
2114
|
const device = getDevice();
|
|
1386
2115
|
const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
|
|
1387
|
-
const disableCommandBatching =
|
|
1388
|
-
|
|
1389
|
-
|
|
2116
|
+
const disableCommandBatching = shouldDisablePrefillCommandBatching(
|
|
2117
|
+
this.#state,
|
|
2118
|
+
opts,
|
|
2119
|
+
multimodalBidirectionalSpan
|
|
2120
|
+
);
|
|
1390
2121
|
const createRecorder = (label) => {
|
|
1391
2122
|
if (!device || disableCommandBatching) return undefined;
|
|
1392
2123
|
return opts.profile ? createProfilingRecorder(label) : createCommandRecorder(label);
|
|
@@ -1405,6 +2136,12 @@ export class PipelineGenerator {
|
|
|
1405
2136
|
opts.executionPlan
|
|
1406
2137
|
);
|
|
1407
2138
|
context.currentTokenIds = inputIds;
|
|
2139
|
+
context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
|
|
2140
|
+
? null
|
|
2141
|
+
: {
|
|
2142
|
+
start: startPos + multimodalBidirectionalSpan.offset,
|
|
2143
|
+
length: multimodalBidirectionalSpan.length,
|
|
2144
|
+
};
|
|
1408
2145
|
let gpuTimePrefillMs = 0;
|
|
1409
2146
|
let hasGpuTimePrefill = false;
|
|
1410
2147
|
const recordProfile = async (rec) => {
|
|
@@ -1416,6 +2153,11 @@ export class PipelineGenerator {
|
|
|
1416
2153
|
hasGpuTimePrefill = true;
|
|
1417
2154
|
}
|
|
1418
2155
|
if (timings) {
|
|
2156
|
+
recordPrefillProfileStep(this.#state, {
|
|
2157
|
+
label: rec.label,
|
|
2158
|
+
timings,
|
|
2159
|
+
totalMs: total ?? undefined,
|
|
2160
|
+
});
|
|
1419
2161
|
log.warn('Profile', `Prefill (${rec.label}):`);
|
|
1420
2162
|
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
1421
2163
|
}
|
|
@@ -1427,9 +2169,23 @@ export class PipelineGenerator {
|
|
|
1427
2169
|
resetSubmitStats();
|
|
1428
2170
|
}
|
|
1429
2171
|
|
|
2172
|
+
const preserveBufferAcrossRecorderSubmit = (buffer, activeRecorder, label) => {
|
|
2173
|
+
if (!activeRecorder || !isGpuBufferInstance(buffer)) {
|
|
2174
|
+
return buffer;
|
|
2175
|
+
}
|
|
2176
|
+
const carryBuffer = acquireBuffer(
|
|
2177
|
+
buffer.size,
|
|
2178
|
+
typeof buffer.usage === 'number' ? buffer.usage : undefined,
|
|
2179
|
+
label
|
|
2180
|
+
);
|
|
2181
|
+
activeRecorder.getEncoder().copyBufferToBuffer(buffer, 0, carryBuffer, 0, buffer.size);
|
|
2182
|
+
activeRecorder.trackTemporaryBuffer(buffer);
|
|
2183
|
+
return carryBuffer;
|
|
2184
|
+
};
|
|
2185
|
+
|
|
1430
2186
|
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
1431
2187
|
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
1432
|
-
let
|
|
2188
|
+
let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
|
|
1433
2189
|
hiddenSize: config.hiddenSize,
|
|
1434
2190
|
vocabSize: config.vocabSize,
|
|
1435
2191
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
@@ -1437,12 +2193,53 @@ export class PipelineGenerator {
|
|
|
1437
2193
|
recorder,
|
|
1438
2194
|
transpose: this.#state.embeddingTranspose,
|
|
1439
2195
|
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
2196
|
+
operatorDiagnostics: this.#state.operatorDiagnostics,
|
|
1440
2197
|
activationDtype,
|
|
1441
2198
|
embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
|
|
2199
|
+
executionPolicies: this.#state.executionV1State?.policies ?? null,
|
|
1442
2200
|
});
|
|
2201
|
+
let hiddenStates = baseEmbeddings;
|
|
2202
|
+
let perLayerInputs = null;
|
|
2203
|
+
try {
|
|
2204
|
+
hiddenStates = await applyPrefixEmbeddingOverride(
|
|
2205
|
+
baseEmbeddings,
|
|
2206
|
+
embeddingOverride,
|
|
2207
|
+
config.hiddenSize,
|
|
2208
|
+
'_prefill',
|
|
2209
|
+
this.#state.executionV1State?.policies ?? null
|
|
2210
|
+
);
|
|
2211
|
+
perLayerInputs = await preparePerLayerInputs(
|
|
2212
|
+
embeddingInputIds,
|
|
2213
|
+
embeddingInputIds === inputIds ? hiddenStates : baseEmbeddings,
|
|
2214
|
+
context,
|
|
2215
|
+
{
|
|
2216
|
+
numTokens,
|
|
2217
|
+
}
|
|
2218
|
+
);
|
|
2219
|
+
} catch (error) {
|
|
2220
|
+
if (isGpuBufferInstance(hiddenStates?.buffer)) {
|
|
2221
|
+
releaseBuffer(hiddenStates.buffer);
|
|
2222
|
+
}
|
|
2223
|
+
if (hiddenStates === baseEmbeddings) {
|
|
2224
|
+
baseEmbeddings = null;
|
|
2225
|
+
}
|
|
2226
|
+
hiddenStates = null;
|
|
2227
|
+
throw error;
|
|
2228
|
+
} finally {
|
|
2229
|
+
if (hiddenStates !== baseEmbeddings && isGpuBufferInstance(baseEmbeddings?.buffer)) {
|
|
2230
|
+
releaseBuffer(baseEmbeddings.buffer);
|
|
2231
|
+
}
|
|
2232
|
+
baseEmbeddings = null;
|
|
2233
|
+
}
|
|
1443
2234
|
|
|
1444
2235
|
if (opts.debug && isGpuBufferInstance(hiddenStates)) {
|
|
1445
2236
|
if (recorder) {
|
|
2237
|
+
hiddenStates = createTensor(
|
|
2238
|
+
preserveBufferAcrossRecorderSubmit(hiddenStates.buffer, recorder, 'prefill_embed_carry'),
|
|
2239
|
+
hiddenStates.dtype,
|
|
2240
|
+
hiddenStates.shape,
|
|
2241
|
+
hiddenStates.label
|
|
2242
|
+
);
|
|
1446
2243
|
await recorder.submitAndWait();
|
|
1447
2244
|
await recordProfile(recorder);
|
|
1448
2245
|
}
|
|
@@ -1473,67 +2270,123 @@ export class PipelineGenerator {
|
|
|
1473
2270
|
|
|
1474
2271
|
let currentRecorder = recorder;
|
|
1475
2272
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
2273
|
+
// Chunked recorder submission: submit every N layers to release tracked intermediate
|
|
2274
|
+
// buffers, preventing unbounded memory growth during large prefills. Critical for
|
|
2275
|
+
// replay_prefill models where each decode step re-runs a prefill-style layer pass.
|
|
2276
|
+
const PREFILL_RECORDER_CHUNK_LAYERS = 4;
|
|
1479
2277
|
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
2278
|
+
let currentHiddenBuffer = hiddenStates.buffer;
|
|
2279
|
+
try {
|
|
2280
|
+
for (let l = 0; l < config.numLayers; l++) {
|
|
2281
|
+
context.recorder = currentRecorder;
|
|
2282
|
+
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
2283
|
+
|
|
2284
|
+
const prevBuffer = currentHiddenBuffer;
|
|
2285
|
+
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
2286
|
+
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
2287
|
+
currentHiddenBuffer = layerOutput;
|
|
2288
|
+
releasePerLayerInputBuffer(
|
|
2289
|
+
context.perLayerInputBuffer,
|
|
2290
|
+
currentRecorder,
|
|
2291
|
+
context.decodeBuffers,
|
|
2292
|
+
this.#state.pleCache ?? null
|
|
2293
|
+
);
|
|
2294
|
+
if (perLayerInputs) {
|
|
2295
|
+
perLayerInputs[l] = null;
|
|
2296
|
+
}
|
|
2297
|
+
context.perLayerInputBuffer = null;
|
|
2298
|
+
|
|
2299
|
+
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
2300
|
+
const isChunkBoundary = !isCheckpoint
|
|
2301
|
+
&& currentRecorder
|
|
2302
|
+
&& l < config.numLayers - 1
|
|
2303
|
+
&& (l + 1) % PREFILL_RECORDER_CHUNK_LAYERS === 0;
|
|
2304
|
+
|
|
2305
|
+
if (isCheckpoint && currentRecorder) {
|
|
2306
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2307
|
+
currentHiddenBuffer,
|
|
2308
|
+
currentRecorder,
|
|
2309
|
+
'prefill_checkpoint_carry'
|
|
2310
|
+
);
|
|
2311
|
+
await currentRecorder.submitAndWait();
|
|
2312
|
+
await recordProfile(currentRecorder);
|
|
2313
|
+
currentRecorder = undefined;
|
|
2314
|
+
}
|
|
1492
2315
|
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
2316
|
+
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
2317
|
+
if (shouldDebug && !currentRecorder) {
|
|
2318
|
+
const device = getDevice();
|
|
2319
|
+
if (device) {
|
|
2320
|
+
if (allowReadback(`pipeline.prefill.layer-${l}`)) {
|
|
2321
|
+
try {
|
|
2322
|
+
const sampleSize = config.hiddenSize * activationBytes;
|
|
2323
|
+
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
2324
|
+
const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
|
|
2325
|
+
const data = decodeReadback(readback, activationDtype);
|
|
2326
|
+
let min = Infinity;
|
|
2327
|
+
let max = -Infinity;
|
|
2328
|
+
let maxAbs = 0;
|
|
2329
|
+
for (const v of data) {
|
|
2330
|
+
if (!Number.isFinite(v)) continue;
|
|
2331
|
+
if (v < min) min = v;
|
|
2332
|
+
if (v > max) max = v;
|
|
2333
|
+
const av = Math.abs(v);
|
|
2334
|
+
if (av > maxAbs) maxAbs = av;
|
|
2335
|
+
}
|
|
2336
|
+
const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
|
|
2337
|
+
log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
|
|
2338
|
+
} catch (e) {
|
|
2339
|
+
log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
|
|
1512
2340
|
}
|
|
1513
|
-
const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
|
|
1514
|
-
log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
|
|
1515
|
-
} catch (e) {
|
|
1516
|
-
log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
|
|
1517
2341
|
}
|
|
1518
2342
|
}
|
|
1519
2343
|
}
|
|
1520
|
-
}
|
|
1521
2344
|
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
2345
|
+
if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
|
|
2346
|
+
currentRecorder = createRecorder('prefill-cont');
|
|
2347
|
+
}
|
|
1525
2348
|
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
2349
|
+
if (prevBuffer !== currentHiddenBuffer) {
|
|
2350
|
+
if (currentRecorder) {
|
|
2351
|
+
currentRecorder.trackTemporaryBuffer(prevBuffer);
|
|
2352
|
+
} else {
|
|
2353
|
+
releaseBuffer(prevBuffer);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
if (isChunkBoundary) {
|
|
2358
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2359
|
+
currentHiddenBuffer,
|
|
2360
|
+
currentRecorder,
|
|
2361
|
+
'prefill_chunk_carry'
|
|
2362
|
+
);
|
|
2363
|
+
await currentRecorder.submitAndWait();
|
|
2364
|
+
await recordProfile(currentRecorder);
|
|
2365
|
+
currentRecorder = createRecorder('prefill-chunk');
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
} finally {
|
|
2369
|
+
context.perLayerInputBuffer = null;
|
|
2370
|
+
if (perLayerInputs) {
|
|
2371
|
+
for (const buffer of perLayerInputs) {
|
|
2372
|
+
releasePerLayerInputBuffer(
|
|
2373
|
+
buffer,
|
|
2374
|
+
currentRecorder,
|
|
2375
|
+
context.decodeBuffers,
|
|
2376
|
+
this.#state.pleCache ?? null
|
|
2377
|
+
);
|
|
1531
2378
|
}
|
|
1532
2379
|
}
|
|
2380
|
+
releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
|
|
1533
2381
|
}
|
|
1534
2382
|
|
|
1535
2383
|
if (this.#state.finitenessBuffer) {
|
|
1536
2384
|
if (currentRecorder) {
|
|
2385
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2386
|
+
currentHiddenBuffer,
|
|
2387
|
+
currentRecorder,
|
|
2388
|
+
'prefill_finiteness_carry'
|
|
2389
|
+
);
|
|
1537
2390
|
await currentRecorder.submitAndWait();
|
|
1538
2391
|
await recordProfile(currentRecorder);
|
|
1539
2392
|
currentRecorder = undefined;
|
|
@@ -1574,6 +2427,13 @@ export class PipelineGenerator {
|
|
|
1574
2427
|
}
|
|
1575
2428
|
|
|
1576
2429
|
if (returnHidden) {
|
|
2430
|
+
if (currentRecorder) {
|
|
2431
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2432
|
+
currentHiddenBuffer,
|
|
2433
|
+
currentRecorder,
|
|
2434
|
+
'prefill_return_hidden_carry'
|
|
2435
|
+
);
|
|
2436
|
+
}
|
|
1577
2437
|
return {
|
|
1578
2438
|
numTokens,
|
|
1579
2439
|
config,
|
|
@@ -1603,7 +2463,8 @@ export class PipelineGenerator {
|
|
|
1603
2463
|
currentHiddenBuffer,
|
|
1604
2464
|
numTokens,
|
|
1605
2465
|
getLogitsWeights(this.#state),
|
|
1606
|
-
getLogitsConfig(this.#state)
|
|
2466
|
+
getLogitsConfig(this.#state),
|
|
2467
|
+
this.#state.operatorDiagnostics
|
|
1607
2468
|
);
|
|
1608
2469
|
logitsVocabSize = recorded.vocabSize;
|
|
1609
2470
|
usedRecordedLogits = true;
|
|
@@ -1636,7 +2497,8 @@ export class PipelineGenerator {
|
|
|
1636
2497
|
undefined,
|
|
1637
2498
|
debugCheckBuffer,
|
|
1638
2499
|
this.#state.runtimeConfig.shared.debug.probes,
|
|
1639
|
-
{ lastPositionOnly: true }
|
|
2500
|
+
{ lastPositionOnly: true },
|
|
2501
|
+
this.#state.operatorDiagnostics
|
|
1640
2502
|
);
|
|
1641
2503
|
const fallbackHealth = getLogitsHealth(fallbackLogits);
|
|
1642
2504
|
if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
|
|
@@ -1655,6 +2517,11 @@ export class PipelineGenerator {
|
|
|
1655
2517
|
releaseBuffer(currentHiddenBuffer);
|
|
1656
2518
|
} else {
|
|
1657
2519
|
if (currentRecorder) {
|
|
2520
|
+
currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
|
|
2521
|
+
currentHiddenBuffer,
|
|
2522
|
+
currentRecorder,
|
|
2523
|
+
'prefill_logits_carry'
|
|
2524
|
+
);
|
|
1658
2525
|
await currentRecorder.submitAndWait();
|
|
1659
2526
|
await recordProfile(currentRecorder);
|
|
1660
2527
|
}
|
|
@@ -1668,7 +2535,8 @@ export class PipelineGenerator {
|
|
|
1668
2535
|
undefined,
|
|
1669
2536
|
debugCheckBuffer,
|
|
1670
2537
|
this.#state.runtimeConfig.shared.debug.probes,
|
|
1671
|
-
{ lastPositionOnly: true }
|
|
2538
|
+
{ lastPositionOnly: true },
|
|
2539
|
+
this.#state.operatorDiagnostics
|
|
1672
2540
|
);
|
|
1673
2541
|
|
|
1674
2542
|
lastLogits = logits.length === logitsVocabSize
|
|
@@ -1708,6 +2576,10 @@ export class PipelineGenerator {
|
|
|
1708
2576
|
|
|
1709
2577
|
|
|
1710
2578
|
async _decodeStep(currentIds, opts) {
|
|
2579
|
+
if (usesReplayPrefillDecode(this.#state)) {
|
|
2580
|
+
const stepResult = await this._decodeStepToLogits(currentIds, opts);
|
|
2581
|
+
return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
|
|
2582
|
+
}
|
|
1711
2583
|
const debugCheckBuffer = this.#state.debug
|
|
1712
2584
|
? (buffer, label, numTokens, expectedDim) =>
|
|
1713
2585
|
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
@@ -1732,6 +2604,7 @@ export class PipelineGenerator {
|
|
|
1732
2604
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1733
2605
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1734
2606
|
resetActiveExecutionPlan(this.#state);
|
|
2607
|
+
assertIncrementalDecodeSupport(this.#state, 'advanceWithToken');
|
|
1735
2608
|
|
|
1736
2609
|
validateCallTimeOptions(options);
|
|
1737
2610
|
|
|
@@ -1749,6 +2622,7 @@ export class PipelineGenerator {
|
|
|
1749
2622
|
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1750
2623
|
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1751
2624
|
resetActiveExecutionPlan(this.#state);
|
|
2625
|
+
assertIncrementalDecodeSupport(this.#state, 'advanceWithTokenAndEmbedding');
|
|
1752
2626
|
|
|
1753
2627
|
validateCallTimeOptions(options);
|
|
1754
2628
|
|