doppler-gpu 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BRANDING.md +14 -0
- package/CHANGELOG.md +158 -0
- package/LICENSE +201 -0
- package/NOTICE +5 -0
- package/README.md +85 -0
- package/SECURITY.md +19 -0
- package/package.json +195 -0
- package/src/adapters/adapter-manager.d.ts +200 -0
- package/src/adapters/adapter-manager.js +509 -0
- package/src/adapters/adapter-manifest.d.ts +290 -0
- package/src/adapters/adapter-manifest.js +320 -0
- package/src/adapters/adapter-registry.d.ts +192 -0
- package/src/adapters/adapter-registry.js +477 -0
- package/src/adapters/index.d.ts +89 -0
- package/src/adapters/index.js +42 -0
- package/src/adapters/lora-loader.d.ts +105 -0
- package/src/adapters/lora-loader.js +425 -0
- package/src/bootstrap.d.ts +1 -0
- package/src/bootstrap.js +30 -0
- package/src/bridge/extension/background.d.ts +14 -0
- package/src/bridge/extension/background.js +168 -0
- package/src/bridge/extension/manifest.json +34 -0
- package/src/bridge/extension-client.d.ts +114 -0
- package/src/bridge/extension-client.js +409 -0
- package/src/bridge/index.d.ts +69 -0
- package/src/bridge/index.js +53 -0
- package/src/bridge/node-bridge.js +26 -0
- package/src/bridge/protocol.d.ts +96 -0
- package/src/bridge/protocol.js +130 -0
- package/src/browser/browser-converter.d.ts +71 -0
- package/src/browser/browser-converter.js +978 -0
- package/src/browser/file-picker.d.ts +63 -0
- package/src/browser/file-picker.js +281 -0
- package/src/browser/gguf-importer.d.ts +136 -0
- package/src/browser/gguf-importer.js +532 -0
- package/src/browser/gguf-parser-browser.d.ts +14 -0
- package/src/browser/gguf-parser-browser.js +17 -0
- package/src/browser/quantization.d.ts +69 -0
- package/src/browser/quantization.js +328 -0
- package/src/browser/safetensors-parser-browser.d.ts +193 -0
- package/src/browser/safetensors-parser-browser.js +347 -0
- package/src/browser/shard-io-browser.d.ts +57 -0
- package/src/browser/shard-io-browser.js +89 -0
- package/src/browser/tensor-source-download.d.ts +27 -0
- package/src/browser/tensor-source-download.js +245 -0
- package/src/browser/tensor-source-file.d.ts +26 -0
- package/src/browser/tensor-source-file.js +53 -0
- package/src/browser/tensor-source-http.d.ts +29 -0
- package/src/browser/tensor-source-http.js +130 -0
- package/src/cli/config/doppler-cli-policy.json +28 -0
- package/src/cli/doppler-cli.d.ts +28 -0
- package/src/cli/doppler-cli.js +1664 -0
- package/src/client/doppler-api.browser.d.ts +1 -0
- package/src/client/doppler-api.browser.js +389 -0
- package/src/client/doppler-api.d.ts +83 -0
- package/src/client/doppler-api.js +389 -0
- package/src/client/doppler-provider/generation.d.ts +25 -0
- package/src/client/doppler-provider/generation.js +126 -0
- package/src/client/doppler-provider/index.d.ts +2 -0
- package/src/client/doppler-provider/index.js +3 -0
- package/src/client/doppler-provider/model-manager.d.ts +71 -0
- package/src/client/doppler-provider/model-manager.js +739 -0
- package/src/client/doppler-provider/provider.d.ts +5 -0
- package/src/client/doppler-provider/provider.js +102 -0
- package/src/client/doppler-provider/source-runtime.d.ts +23 -0
- package/src/client/doppler-provider/source-runtime.js +677 -0
- package/src/client/doppler-provider/types.d.ts +127 -0
- package/src/client/doppler-provider/types.js +17 -0
- package/src/client/doppler-provider.d.ts +46 -0
- package/src/client/doppler-provider.js +36 -0
- package/src/client/doppler-registry.d.ts +23 -0
- package/src/client/doppler-registry.js +86 -0
- package/src/client/doppler-registry.json +41 -0
- package/src/config/README.md +57 -0
- package/src/config/backward-registry-loader.d.ts +3 -0
- package/src/config/backward-registry-loader.js +23 -0
- package/src/config/conversion/README.md +147 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +285 -0
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +321 -0
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +322 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +322 -0
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +322 -0
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +323 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +158 -0
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +164 -0
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +158 -0
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +339 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +385 -0
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +385 -0
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +219 -0
- package/src/config/execution-contract-check.d.ts +72 -0
- package/src/config/execution-contract-check.js +290 -0
- package/src/config/index.d.ts +52 -0
- package/src/config/index.js +22 -0
- package/src/config/kernel-path-loader.d.ts +156 -0
- package/src/config/kernel-path-loader.js +396 -0
- package/src/config/kernels/backward-registry.json +99 -0
- package/src/config/kernels/kernel-ref-digests.d.ts +1 -0
- package/src/config/kernels/kernel-ref-digests.js +233 -0
- package/src/config/kernels/kernel-ref.d.ts +17 -0
- package/src/config/kernels/kernel-ref.js +75 -0
- package/src/config/kernels/moe/gpt-oss.paths.json +49 -0
- package/src/config/kernels/moe/mixtral.paths.json +46 -0
- package/src/config/kernels/registry.d.ts +86 -0
- package/src/config/kernels/registry.js +116 -0
- package/src/config/kernels/registry.json +9446 -0
- package/src/config/merge-contract-check.d.ts +16 -0
- package/src/config/merge-contract-check.js +311 -0
- package/src/config/merge-helpers.d.ts +58 -0
- package/src/config/merge-helpers.js +179 -0
- package/src/config/merge.d.ts +153 -0
- package/src/config/merge.js +445 -0
- package/src/config/param-categories.d.ts +17 -0
- package/src/config/param-categories.js +72 -0
- package/src/config/param-validator.d.ts +26 -0
- package/src/config/param-validator.js +274 -0
- package/src/config/platforms/amd-rdna3.json +16 -0
- package/src/config/platforms/apple-m1.json +16 -0
- package/src/config/platforms/apple-m2.json +16 -0
- package/src/config/platforms/apple-m3.json +16 -0
- package/src/config/platforms/generic.json +14 -0
- package/src/config/platforms/loader.d.ts +65 -0
- package/src/config/platforms/loader.js +158 -0
- package/src/config/platforms/nvidia-gb200-8gpu.json +45 -0
- package/src/config/platforms/nvidia-gb200-nvl72.json +45 -0
- package/src/config/platforms/nvidia-gh200-nvl2.json +44 -0
- package/src/config/platforms/nvidia-gh200.json +44 -0
- package/src/config/platforms/nvidia-rtx30.json +16 -0
- package/src/config/platforms/nvidia-rtx40.json +16 -0
- package/src/config/quantization-contract-check.d.ts +12 -0
- package/src/config/quantization-contract-check.js +91 -0
- package/src/config/required-inference-fields-contract-check.d.ts +24 -0
- package/src/config/required-inference-fields-contract-check.js +238 -0
- package/src/config/runtime/compute/f16-activations.json +30 -0
- package/src/config/runtime/compute/f16-batched.json +32 -0
- package/src/config/runtime/default.json +100 -0
- package/src/config/runtime/diagnostics/debug-logits.json +53 -0
- package/src/config/runtime/diagnostics/drift-policies.json +147 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +54 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +210 -0
- package/src/config/runtime/experiments/verify/gemma3-verify.json +39 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +46 -0
- package/src/config/runtime/experiments/verify/translategemma-verify.json +39 -0
- package/src/config/runtime/kernels/dequant-f16-q4k.json +13 -0
- package/src/config/runtime/kernels/dequant-f32-q4k.json +13 -0
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +37 -0
- package/src/config/runtime/kernels/fused-q4k.json +13 -0
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +33 -0
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +33 -0
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +33 -0
- package/src/config/runtime/kernels/safe-q4k.json +13 -0
- package/src/config/runtime/model/gemma2-debug.json +77 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +66 -0
- package/src/config/runtime/model/gemma2-pipeline.json +75 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +85 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +52 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +108 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +90 -0
- package/src/config/runtime/overlays/debug-f32a.json +3 -0
- package/src/config/runtime/overlays/trace-config.json +28 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +11 -0
- package/src/config/runtime/profiles/default.json +10 -0
- package/src/config/runtime/profiles/low-memory.json +40 -0
- package/src/config/runtime/profiles/production.json +48 -0
- package/src/config/runtime/profiles/simulation.json +31 -0
- package/src/config/runtime/profiles/throughput.json +37 -0
- package/src/config/runtime/profiles/trace-layers.json +127 -0
- package/src/config/runtime/profiles/vector-stability.json +54 -0
- package/src/config/runtime/profiles/vector-throughput.json +28 -0
- package/src/config/runtime/profiles/verbose-trace.json +39 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +69 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +66 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +66 -0
- package/src/config/runtime-merge.d.ts +5 -0
- package/src/config/runtime-merge.js +30 -0
- package/src/config/runtime.d.ts +28 -0
- package/src/config/runtime.js +69 -0
- package/src/config/schema/adapter.schema.d.ts +53 -0
- package/src/config/schema/adapter.schema.js +60 -0
- package/src/config/schema/backward-registry.schema.d.ts +14 -0
- package/src/config/schema/backward-registry.schema.js +46 -0
- package/src/config/schema/benchmark.schema.d.ts +54 -0
- package/src/config/schema/benchmark.schema.js +74 -0
- package/src/config/schema/bridge.schema.d.ts +25 -0
- package/src/config/schema/bridge.schema.js +22 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.js +44 -0
- package/src/config/schema/buffer-pool.schema.d.ts +92 -0
- package/src/config/schema/buffer-pool.schema.js +50 -0
- package/src/config/schema/conversion-report.schema.d.ts +36 -0
- package/src/config/schema/conversion-report.schema.js +97 -0
- package/src/config/schema/conversion.schema.d.ts +191 -0
- package/src/config/schema/conversion.schema.js +13 -0
- package/src/config/schema/converter.schema.d.ts +117 -0
- package/src/config/schema/converter.schema.js +140 -0
- package/src/config/schema/debug.schema.d.ts +292 -0
- package/src/config/schema/debug.schema.js +134 -0
- package/src/config/schema/diffusion.schema.d.ts +88 -0
- package/src/config/schema/diffusion.schema.js +62 -0
- package/src/config/schema/distill-training.schema.d.ts +48 -0
- package/src/config/schema/distill-training.schema.js +139 -0
- package/src/config/schema/distribution.schema.d.ts +155 -0
- package/src/config/schema/distribution.schema.js +81 -0
- package/src/config/schema/doppler.schema.d.ts +74 -0
- package/src/config/schema/doppler.schema.js +341 -0
- package/src/config/schema/ecosystem.schema.d.ts +255 -0
- package/src/config/schema/ecosystem.schema.js +534 -0
- package/src/config/schema/emulation.schema.d.ts +351 -0
- package/src/config/schema/emulation.schema.js +301 -0
- package/src/config/schema/energy.schema.d.ts +102 -0
- package/src/config/schema/energy.schema.js +72 -0
- package/src/config/schema/execution-v1.schema.d.ts +258 -0
- package/src/config/schema/execution-v1.schema.js +211 -0
- package/src/config/schema/gpu-cache.schema.d.ts +26 -0
- package/src/config/schema/gpu-cache.schema.js +8 -0
- package/src/config/schema/harness.schema.d.ts +34 -0
- package/src/config/schema/harness.schema.js +21 -0
- package/src/config/schema/hotswap.schema.d.ts +55 -0
- package/src/config/schema/hotswap.schema.js +18 -0
- package/src/config/schema/index.d.ts +869 -0
- package/src/config/schema/index.js +498 -0
- package/src/config/schema/inference-defaults.schema.d.ts +276 -0
- package/src/config/schema/inference-defaults.schema.js +189 -0
- package/src/config/schema/inference.schema.d.ts +298 -0
- package/src/config/schema/inference.schema.js +39 -0
- package/src/config/schema/intent-bundle.schema.d.ts +28 -0
- package/src/config/schema/intent-bundle.schema.js +12 -0
- package/src/config/schema/kernel-path.schema.d.ts +184 -0
- package/src/config/schema/kernel-path.schema.js +9 -0
- package/src/config/schema/kernel-registry.schema.d.ts +199 -0
- package/src/config/schema/kernel-registry.schema.js +46 -0
- package/src/config/schema/kernel-thresholds.schema.d.ts +302 -0
- package/src/config/schema/kernel-thresholds.schema.js +195 -0
- package/src/config/schema/kernel-warmup.schema.d.ts +19 -0
- package/src/config/schema/kernel-warmup.schema.js +5 -0
- package/src/config/schema/kvcache.schema.d.ts +131 -0
- package/src/config/schema/kvcache.schema.js +41 -0
- package/src/config/schema/loading.schema.d.ts +153 -0
- package/src/config/schema/loading.schema.js +84 -0
- package/src/config/schema/lora.schema.d.ts +12 -0
- package/src/config/schema/lora.schema.js +12 -0
- package/src/config/schema/manifest.schema.d.ts +520 -0
- package/src/config/schema/manifest.schema.js +145 -0
- package/src/config/schema/memory-limits.schema.d.ts +107 -0
- package/src/config/schema/memory-limits.schema.js +57 -0
- package/src/config/schema/moe.schema.d.ts +78 -0
- package/src/config/schema/moe.schema.js +40 -0
- package/src/config/schema/platform.schema.d.ts +121 -0
- package/src/config/schema/platform.schema.js +1 -0
- package/src/config/schema/quantization-defaults.schema.d.ts +34 -0
- package/src/config/schema/quantization-defaults.schema.js +5 -0
- package/src/config/schema/quantization.schema.d.ts +10 -0
- package/src/config/schema/quantization.schema.js +33 -0
- package/src/config/schema/shared-runtime.schema.d.ts +75 -0
- package/src/config/schema/shared-runtime.schema.js +45 -0
- package/src/config/schema/speculation-self.schema.js +54 -0
- package/src/config/schema/speculative.schema.d.ts +21 -0
- package/src/config/schema/speculative.schema.js +11 -0
- package/src/config/schema/storage.schema.d.ts +123 -0
- package/src/config/schema/storage.schema.js +66 -0
- package/src/config/schema/tooling.schema.d.ts +29 -0
- package/src/config/schema/tooling.schema.js +12 -0
- package/src/config/schema/training-metrics.schema.d.ts +89 -0
- package/src/config/schema/training-metrics.schema.js +374 -0
- package/src/config/schema/training.schema.d.ts +88 -0
- package/src/config/schema/training.schema.js +106 -0
- package/src/config/schema/tuner.schema.d.ts +39 -0
- package/src/config/schema/tuner.schema.js +13 -0
- package/src/config/schema/ul-training.schema.d.ts +61 -0
- package/src/config/schema/ul-training.schema.js +140 -0
- package/src/config/schema/units.schema.d.ts +27 -0
- package/src/config/schema/units.schema.js +30 -0
- package/src/config/training-defaults.d.ts +24 -0
- package/src/config/training-defaults.js +99 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +34 -0
- package/src/config/transforms/capability-transform-resolver.js +62 -0
- package/src/config/transforms/execution-graph-transforms.d.ts +92 -0
- package/src/config/transforms/execution-graph-transforms.js +512 -0
- package/src/converter/conversion-plan.d.ts +54 -0
- package/src/converter/conversion-plan.js +268 -0
- package/src/converter/core.d.ts +263 -0
- package/src/converter/core.js +1530 -0
- package/src/converter/index.d.ts +98 -0
- package/src/converter/index.js +59 -0
- package/src/converter/parsers/diffusion.d.ts +50 -0
- package/src/converter/parsers/diffusion.js +327 -0
- package/src/converter/parsers/gguf.d.ts +22 -0
- package/src/converter/parsers/gguf.js +46 -0
- package/src/converter/parsers/index.d.ts +21 -0
- package/src/converter/parsers/index.js +12 -0
- package/src/converter/parsers/transformer.d.ts +31 -0
- package/src/converter/parsers/transformer.js +198 -0
- package/src/converter/quantization-info.d.ts +37 -0
- package/src/converter/quantization-info.js +422 -0
- package/src/converter/quantizer.d.ts +101 -0
- package/src/converter/quantizer.js +444 -0
- package/src/converter/rope-config.d.ts +15 -0
- package/src/converter/rope-config.js +262 -0
- package/src/converter/shard-packer.d.ts +138 -0
- package/src/converter/shard-packer.js +425 -0
- package/src/converter/tokenizer-utils.d.ts +12 -0
- package/src/converter/tokenizer-utils.js +104 -0
- package/src/debug/capture-policy.d.ts +88 -0
- package/src/debug/capture-policy.js +196 -0
- package/src/debug/config.d.ts +78 -0
- package/src/debug/config.js +347 -0
- package/src/debug/history.d.ts +65 -0
- package/src/debug/history.js +82 -0
- package/src/debug/index.d.ts +268 -0
- package/src/debug/index.js +215 -0
- package/src/debug/log.d.ts +46 -0
- package/src/debug/log.js +132 -0
- package/src/debug/perf.d.ts +33 -0
- package/src/debug/perf.js +51 -0
- package/src/debug/reference/README.md +114 -0
- package/src/debug/reference/hf_attn_debug.py +114 -0
- package/src/debug/reference/hf_embed_check.py +89 -0
- package/src/debug/reference/hf_layer_out.py +100 -0
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +268 -0
- package/src/debug/reference/hf_rope_check.py +116 -0
- package/src/debug/reference/hf_weights.py +75 -0
- package/src/debug/signals.d.ts +63 -0
- package/src/debug/signals.js +39 -0
- package/src/debug/stats.d.ts +47 -0
- package/src/debug/stats.js +160 -0
- package/src/debug/tensor.d.ts +125 -0
- package/src/debug/tensor.js +268 -0
- package/src/debug/trace.d.ts +17 -0
- package/src/debug/trace.js +188 -0
- package/src/diffusion/config/expansion-lanes.json +29 -0
- package/src/diffusion/image-regression.d.ts +31 -0
- package/src/diffusion/image-regression.js +107 -0
- package/src/diffusion/index.d.ts +8 -0
- package/src/diffusion/index.js +8 -0
- package/src/distribution/p2p-control-plane.d.ts +52 -0
- package/src/distribution/p2p-control-plane.js +272 -0
- package/src/distribution/p2p-observability.d.ts +116 -0
- package/src/distribution/p2p-observability.js +303 -0
- package/src/distribution/p2p-transport-contract.d.ts +57 -0
- package/src/distribution/p2p-transport-contract.js +310 -0
- package/src/distribution/p2p-transport.js +75 -0
- package/src/distribution/p2p-webrtc-browser.d.ts +37 -0
- package/src/distribution/p2p-webrtc-browser.js +454 -0
- package/src/distribution/shard-delivery.d.ts +251 -0
- package/src/distribution/shard-delivery.js +2186 -0
- package/src/energy/index.d.ts +2 -0
- package/src/energy/index.js +2 -0
- package/src/errors/doppler-error.d.ts +20 -0
- package/src/errors/doppler-error.js +37 -0
- package/src/errors/index.d.ts +1 -0
- package/src/errors/index.js +1 -0
- package/src/formats/gguf/index.d.ts +8 -0
- package/src/formats/gguf/index.js +4 -0
- package/src/formats/gguf/types.d.ts +137 -0
- package/src/formats/gguf/types.js +460 -0
- package/src/formats/index.d.ts +51 -0
- package/src/formats/index.js +13 -0
- package/src/formats/rdrr/classification.d.ts +54 -0
- package/src/formats/rdrr/classification.js +329 -0
- package/src/formats/rdrr/groups.d.ts +35 -0
- package/src/formats/rdrr/groups.js +73 -0
- package/src/formats/rdrr/index.d.ts +25 -0
- package/src/formats/rdrr/index.js +19 -0
- package/src/formats/rdrr/manifest.d.ts +32 -0
- package/src/formats/rdrr/manifest.js +108 -0
- package/src/formats/rdrr/parsing.d.ts +27 -0
- package/src/formats/rdrr/parsing.js +167 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +42 -0
- package/src/formats/rdrr/tensor-config-validator.js +156 -0
- package/src/formats/rdrr/types.d.ts +201 -0
- package/src/formats/rdrr/types.js +16 -0
- package/src/formats/rdrr/validation.d.ts +9 -0
- package/src/formats/rdrr/validation.js +213 -0
- package/src/formats/safetensors/index.d.ts +8 -0
- package/src/formats/safetensors/index.js +4 -0
- package/src/formats/safetensors/types.d.ts +67 -0
- package/src/formats/safetensors/types.js +102 -0
- package/src/formats/tokenizer/index.d.ts +5 -0
- package/src/formats/tokenizer/index.js +3 -0
- package/src/formats/tokenizer/types.d.ts +9 -0
- package/src/formats/tokenizer/types.js +22 -0
- package/src/generation/index.d.ts +18 -0
- package/src/generation/index.js +32 -0
- package/src/gpu/command-recorder.d.ts +175 -0
- package/src/gpu/command-recorder.js +498 -0
- package/src/gpu/device.d.ts +142 -0
- package/src/gpu/device.js +532 -0
- package/src/gpu/kernel-runtime.d.ts +20 -0
- package/src/gpu/kernel-runtime.js +39 -0
- package/src/gpu/kernel-selection-cache.d.ts +13 -0
- package/src/gpu/kernel-selection-cache.js +13 -0
- package/src/gpu/kernel-selection-log.d.ts +12 -0
- package/src/gpu/kernel-selection-log.js +28 -0
- package/src/gpu/kernel-selector.d.ts +11 -0
- package/src/gpu/kernel-selector.js +10 -0
- package/src/gpu/kernel-tuner/benchmarks.d.ts +144 -0
- package/src/gpu/kernel-tuner/benchmarks.js +902 -0
- package/src/gpu/kernel-tuner/cache.d.ts +55 -0
- package/src/gpu/kernel-tuner/cache.js +152 -0
- package/src/gpu/kernel-tuner/index.d.ts +59 -0
- package/src/gpu/kernel-tuner/index.js +38 -0
- package/src/gpu/kernel-tuner/tuner.d.ts +82 -0
- package/src/gpu/kernel-tuner/tuner.js +247 -0
- package/src/gpu/kernel-tuner/types.d.ts +101 -0
- package/src/gpu/kernel-tuner/types.js +4 -0
- package/src/gpu/kernel-tuner.d.ts +33 -0
- package/src/gpu/kernel-tuner.js +12 -0
- package/src/gpu/kernels/README.md +127 -0
- package/src/gpu/kernels/attention.d.ts +236 -0
- package/src/gpu/kernels/attention.js +1437 -0
- package/src/gpu/kernels/attention.wgsl +249 -0
- package/src/gpu/kernels/attention_bdpa_decode_f16.wgsl +246 -0
- package/src/gpu/kernels/attention_decode.wgsl +233 -0
- package/src/gpu/kernels/attention_decode_chunked_f16.wgsl +183 -0
- package/src/gpu/kernels/attention_decode_chunked_f16kv.wgsl +208 -0
- package/src/gpu/kernels/attention_decode_f16.wgsl +202 -0
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +224 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +223 -0
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +225 -0
- package/src/gpu/kernels/attention_decode_optimized.wgsl +445 -0
- package/src/gpu/kernels/attention_decode_paged_f16.wgsl +172 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +174 -0
- package/src/gpu/kernels/attention_decode_subgroup.wgsl +233 -0
- package/src/gpu/kernels/attention_decode_tiered_f16.wgsl +218 -0
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +220 -0
- package/src/gpu/kernels/attention_decode_tiered_int4_f16kv.wgsl +242 -0
- package/src/gpu/kernels/attention_decode_tiered_int8_f16kv.wgsl +242 -0
- package/src/gpu/kernels/attention_f16.wgsl +214 -0
- package/src/gpu/kernels/attention_f16kv.wgsl +242 -0
- package/src/gpu/kernels/attention_small.wgsl +260 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +240 -0
- package/src/gpu/kernels/attention_small_f16kv.wgsl +266 -0
- package/src/gpu/kernels/attention_streaming.wgsl +149 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +147 -0
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +151 -0
- package/src/gpu/kernels/backward/adam.d.ts +28 -0
- package/src/gpu/kernels/backward/adam.js +203 -0
- package/src/gpu/kernels/backward/adam.wgsl +50 -0
- package/src/gpu/kernels/backward/attention_backward.d.ts +22 -0
- package/src/gpu/kernels/backward/attention_backward.js +364 -0
- package/src/gpu/kernels/backward/attention_backward.wgsl +49 -0
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +17 -0
- package/src/gpu/kernels/backward/bias_add_backward.js +24 -0
- package/src/gpu/kernels/backward/bias_add_backward.wgsl +33 -0
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +31 -0
- package/src/gpu/kernels/backward/conv2d_backward.js +148 -0
- package/src/gpu/kernels/backward/conv2d_backward_input.wgsl +83 -0
- package/src/gpu/kernels/backward/conv2d_backward_weight.wgsl +70 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +23 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.js +29 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +39 -0
- package/src/gpu/kernels/backward/embed_backward.d.ts +29 -0
- package/src/gpu/kernels/backward/embed_backward.js +118 -0
- package/src/gpu/kernels/backward/embed_backward.wgsl +73 -0
- package/src/gpu/kernels/backward/gelu_backward.d.ts +16 -0
- package/src/gpu/kernels/backward/gelu_backward.js +39 -0
- package/src/gpu/kernels/backward/gelu_backward.wgsl +38 -0
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/groupnorm_backward.js +29 -0
- package/src/gpu/kernels/backward/groupnorm_backward.wgsl +143 -0
- package/src/gpu/kernels/backward/index.d.ts +17 -0
- package/src/gpu/kernels/backward/index.js +23 -0
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +22 -0
- package/src/gpu/kernels/backward/layernorm_backward.js +135 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +194 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/matmul_backward.js +124 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +90 -0
- package/src/gpu/kernels/backward/matmul_transpose_a.wgsl +84 -0
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +22 -0
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +30 -0
- package/src/gpu/kernels/backward/pixel_shuffle_backward.wgsl +54 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +101 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +78 -0
- package/src/gpu/kernels/backward/rope_backward.d.ts +25 -0
- package/src/gpu/kernels/backward/rope_backward.js +109 -0
- package/src/gpu/kernels/backward/rope_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/scale_backward.d.ts +16 -0
- package/src/gpu/kernels/backward/scale_backward.js +84 -0
- package/src/gpu/kernels/backward/scale_backward.wgsl +27 -0
- package/src/gpu/kernels/backward/silu_backward.d.ts +16 -0
- package/src/gpu/kernels/backward/silu_backward.js +39 -0
- package/src/gpu/kernels/backward/silu_backward.wgsl +31 -0
- package/src/gpu/kernels/backward/softmax_backward.d.ts +16 -0
- package/src/gpu/kernels/backward/softmax_backward.js +43 -0
- package/src/gpu/kernels/backward/softmax_backward.wgsl +44 -0
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +21 -0
- package/src/gpu/kernels/backward/upsample2d_backward.js +30 -0
- package/src/gpu/kernels/backward/upsample2d_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/utils.d.ts +45 -0
- package/src/gpu/kernels/backward/utils.js +371 -0
- package/src/gpu/kernels/bf16_to_f16.wgsl +54 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +70 -0
- package/src/gpu/kernels/bias_add.wgsl +42 -0
- package/src/gpu/kernels/bias_add_f16.wgsl +47 -0
- package/src/gpu/kernels/cast.d.ts +67 -0
- package/src/gpu/kernels/cast.js +339 -0
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +31 -0
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +36 -0
- package/src/gpu/kernels/check-finiteness.d.ts +15 -0
- package/src/gpu/kernels/check-finiteness.js +149 -0
- package/src/gpu/kernels/check-stop.d.ts +31 -0
- package/src/gpu/kernels/check-stop.js +170 -0
- package/src/gpu/kernels/clamp.d.ts +22 -0
- package/src/gpu/kernels/clamp.js +42 -0
- package/src/gpu/kernels/clamp.wgsl +24 -0
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +213 -0
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +89 -0
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +258 -0
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +114 -0
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +193 -0
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +264 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +109 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +53 -0
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +54 -0
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +125 -0
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +428 -0
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +114 -0
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +477 -0
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +60 -0
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +58 -0
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +82 -0
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +87 -0
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +181 -0
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +62 -0
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +430 -0
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +462 -0
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +307 -0
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +65 -0
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +81 -0
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +231 -0
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +152 -0
- package/src/gpu/kernels/codegen/wgsl-variants.js +733 -0
- package/src/gpu/kernels/constants.d.ts +168 -0
- package/src/gpu/kernels/constants.js +142 -0
- package/src/gpu/kernels/conv2d.d.ts +34 -0
- package/src/gpu/kernels/conv2d.js +91 -0
- package/src/gpu/kernels/conv2d.wgsl +70 -0
- package/src/gpu/kernels/conv2d_f16.wgsl +72 -0
- package/src/gpu/kernels/cross_entropy_loss.d.ts +21 -0
- package/src/gpu/kernels/cross_entropy_loss.js +60 -0
- package/src/gpu/kernels/cross_entropy_loss.wgsl +39 -0
- package/src/gpu/kernels/depthwise_conv2d.d.ts +29 -0
- package/src/gpu/kernels/depthwise_conv2d.js +109 -0
- package/src/gpu/kernels/depthwise_conv2d.wgsl +55 -0
- package/src/gpu/kernels/depthwise_conv2d_f16.wgsl +59 -0
- package/src/gpu/kernels/dequant.d.ts +108 -0
- package/src/gpu/kernels/dequant.js +575 -0
- package/src/gpu/kernels/dequant_f16_out.wgsl +153 -0
- package/src/gpu/kernels/dequant_f16_out_vec4.wgsl +152 -0
- package/src/gpu/kernels/dequant_f16_rowwise.wgsl +139 -0
- package/src/gpu/kernels/dequant_f32_rowwise.wgsl +133 -0
- package/src/gpu/kernels/dequant_mxfp4.wgsl +120 -0
- package/src/gpu/kernels/dequant_mxfp4_expert.wgsl +129 -0
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +105 -0
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +116 -0
- package/src/gpu/kernels/dequant_q6k.wgsl +140 -0
- package/src/gpu/kernels/dequant_q8_0.wgsl +98 -0
- package/src/gpu/kernels/dequant_shared.wgsl +200 -0
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +160 -0
- package/src/gpu/kernels/dequant_subgroup.wgsl +206 -0
- package/src/gpu/kernels/dispatch.d.ts +157 -0
- package/src/gpu/kernels/dispatch.js +235 -0
- package/src/gpu/kernels/energy.d.ts +113 -0
- package/src/gpu/kernels/energy.js +448 -0
- package/src/gpu/kernels/energy_eval.wgsl +26 -0
- package/src/gpu/kernels/energy_eval_f16.wgsl +30 -0
- package/src/gpu/kernels/energy_quintel_grad.wgsl +92 -0
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +96 -0
- package/src/gpu/kernels/energy_quintel_reduce.wgsl +112 -0
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +116 -0
- package/src/gpu/kernels/energy_quintel_update.wgsl +92 -0
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +96 -0
- package/src/gpu/kernels/energy_update.wgsl +25 -0
- package/src/gpu/kernels/energy_update_f16.wgsl +30 -0
- package/src/gpu/kernels/feature-check.d.ts +42 -0
- package/src/gpu/kernels/feature-check.js +70 -0
- package/src/gpu/kernels/fused_ffn.d.ts +65 -0
- package/src/gpu/kernels/fused_ffn.js +326 -0
- package/src/gpu/kernels/fused_ffn.wgsl +420 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +213 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +375 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +404 -0
- package/src/gpu/kernels/fused_matmul_q4_batched.wgsl +194 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +170 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +154 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +219 -0
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +216 -0
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +204 -0
- package/src/gpu/kernels/fused_matmul_residual.d.ts +46 -0
- package/src/gpu/kernels/fused_matmul_residual.js +175 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.d.ts +64 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +290 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.wgsl +324 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +303 -0
- package/src/gpu/kernels/fused_swiglu.wgsl +63 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +57 -0
- package/src/gpu/kernels/gated-short-conv.d.ts +63 -0
- package/src/gpu/kernels/gated-short-conv.js +284 -0
- package/src/gpu/kernels/gather.d.ts +64 -0
- package/src/gpu/kernels/gather.js +137 -0
- package/src/gpu/kernels/gather.wgsl +61 -0
- package/src/gpu/kernels/gather_f16.wgsl +65 -0
- package/src/gpu/kernels/gather_f16_f16_out.wgsl +55 -0
- package/src/gpu/kernels/gather_f16_out.wgsl +55 -0
- package/src/gpu/kernels/gather_f16_vec4.wgsl +76 -0
- package/src/gpu/kernels/gather_f16_vec4_f16_out.wgsl +68 -0
- package/src/gpu/kernels/gather_vec4.wgsl +74 -0
- package/src/gpu/kernels/gather_vec4_f16_out.wgsl +68 -0
- package/src/gpu/kernels/gelu.d.ts +33 -0
- package/src/gpu/kernels/gelu.js +55 -0
- package/src/gpu/kernels/gelu.wgsl +64 -0
- package/src/gpu/kernels/gelu_f16.wgsl +66 -0
- package/src/gpu/kernels/gptoss_mxfp4_expert_fused.wgsl +127 -0
- package/src/gpu/kernels/gptoss_router_topk.wgsl +119 -0
- package/src/gpu/kernels/grouped_pointwise_conv2d.d.ts +27 -0
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +103 -0
- package/src/gpu/kernels/grouped_pointwise_conv2d.wgsl +44 -0
- package/src/gpu/kernels/grouped_pointwise_conv2d_f16.wgsl +48 -0
- package/src/gpu/kernels/groupnorm.d.ts +31 -0
- package/src/gpu/kernels/groupnorm.js +102 -0
- package/src/gpu/kernels/groupnorm_apply.wgsl +41 -0
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +46 -0
- package/src/gpu/kernels/groupnorm_stats.wgsl +76 -0
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +79 -0
- package/src/gpu/kernels/index.d.ts +374 -0
- package/src/gpu/kernels/index.js +315 -0
- package/src/gpu/kernels/kernel-base.d.ts +33 -0
- package/src/gpu/kernels/kernel-base.js +46 -0
- package/src/gpu/kernels/kernel-configs.d.ts +65 -0
- package/src/gpu/kernels/kernel-configs.js +60 -0
- package/src/gpu/kernels/kernel-tuning.d.ts +42 -0
- package/src/gpu/kernels/kernel-tuning.js +158 -0
- package/src/gpu/kernels/kv-quantize.d.ts +37 -0
- package/src/gpu/kernels/kv-quantize.js +141 -0
- package/src/gpu/kernels/kv_quantize_int4.wgsl +119 -0
- package/src/gpu/kernels/kv_quantize_int8.wgsl +119 -0
- package/src/gpu/kernels/layernorm.d.ts +37 -0
- package/src/gpu/kernels/layernorm.js +96 -0
- package/src/gpu/kernels/layernorm.wgsl +121 -0
- package/src/gpu/kernels/layernorm_f16.wgsl +103 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +39 -0
- package/src/gpu/kernels/linear-attention-core.js +559 -0
- package/src/gpu/kernels/logit-merge.d.ts +110 -0
- package/src/gpu/kernels/logit-merge.js +394 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +38 -0
- package/src/gpu/kernels/matmul-dispatch.js +156 -0
- package/src/gpu/kernels/matmul-selection.d.ts +87 -0
- package/src/gpu/kernels/matmul-selection.js +518 -0
- package/src/gpu/kernels/matmul.d.ts +114 -0
- package/src/gpu/kernels/matmul.js +384 -0
- package/src/gpu/kernels/matmul_f16.wgsl +170 -0
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +165 -0
- package/src/gpu/kernels/matmul_f16w_f32a.wgsl +89 -0
- package/src/gpu/kernels/matmul_f16w_f32a_tiled.wgsl +154 -0
- package/src/gpu/kernels/matmul_f32.wgsl +100 -0
- package/src/gpu/kernels/matmul_gemv.wgsl +80 -0
- package/src/gpu/kernels/matmul_gemv_f16a.wgsl +81 -0
- package/src/gpu/kernels/matmul_gemv_residual.wgsl +119 -0
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +78 -0
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +429 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +674 -0
- package/src/gpu/kernels/modulate.d.ts +29 -0
- package/src/gpu/kernels/modulate.js +57 -0
- package/src/gpu/kernels/modulate.wgsl +40 -0
- package/src/gpu/kernels/modulate_f16.wgsl +43 -0
- package/src/gpu/kernels/moe.d.ts +164 -0
- package/src/gpu/kernels/moe.js +542 -0
- package/src/gpu/kernels/moe_gather.wgsl +170 -0
- package/src/gpu/kernels/moe_gather_f16.wgsl +82 -0
- package/src/gpu/kernels/moe_gather_vec4.wgsl +74 -0
- package/src/gpu/kernels/moe_offsets.wgsl +48 -0
- package/src/gpu/kernels/pipeline-cache.d.ts +88 -0
- package/src/gpu/kernels/pipeline-cache.js +305 -0
- package/src/gpu/kernels/pixel_shuffle.d.ts +27 -0
- package/src/gpu/kernels/pixel_shuffle.js +57 -0
- package/src/gpu/kernels/pixel_shuffle.wgsl +43 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +46 -0
- package/src/gpu/kernels/relu.d.ts +18 -0
- package/src/gpu/kernels/relu.js +66 -0
- package/src/gpu/kernels/relu.wgsl +22 -0
- package/src/gpu/kernels/relu_f16.wgsl +24 -0
- package/src/gpu/kernels/rep-penalty.js +103 -0
- package/src/gpu/kernels/rep_penalty.wgsl +57 -0
- package/src/gpu/kernels/rep_penalty_f16.wgsl +54 -0
- package/src/gpu/kernels/repeat_channels.d.ts +21 -0
- package/src/gpu/kernels/repeat_channels.js +68 -0
- package/src/gpu/kernels/repeat_channels.wgsl +28 -0
- package/src/gpu/kernels/repeat_channels_f16.wgsl +30 -0
- package/src/gpu/kernels/residual.d.ts +74 -0
- package/src/gpu/kernels/residual.js +173 -0
- package/src/gpu/kernels/residual.wgsl +56 -0
- package/src/gpu/kernels/residual_f16.wgsl +36 -0
- package/src/gpu/kernels/residual_f16_vec4.wgsl +48 -0
- package/src/gpu/kernels/residual_vec4.wgsl +47 -0
- package/src/gpu/kernels/rmsnorm.d.ts +53 -0
- package/src/gpu/kernels/rmsnorm.js +256 -0
- package/src/gpu/kernels/rmsnorm.wgsl +449 -0
- package/src/gpu/kernels/rmsnorm_f16.wgsl +198 -0
- package/src/gpu/kernels/rope.d.ts +50 -0
- package/src/gpu/kernels/rope.js +91 -0
- package/src/gpu/kernels/rope.wgsl +344 -0
- package/src/gpu/kernels/rope_f16.wgsl +271 -0
- package/src/gpu/kernels/rule-matcher.d.ts +30 -0
- package/src/gpu/kernels/rule-matcher.js +42 -0
- package/src/gpu/kernels/rule-registry.d.ts +7 -0
- package/src/gpu/kernels/rule-registry.js +54 -0
- package/src/gpu/kernels/sample.d.ts +75 -0
- package/src/gpu/kernels/sample.js +565 -0
- package/src/gpu/kernels/sample.wgsl +407 -0
- package/src/gpu/kernels/sample_f16.wgsl +363 -0
- package/src/gpu/kernels/sana_linear_attention.d.ts +27 -0
- package/src/gpu/kernels/sana_linear_attention.js +129 -0
- package/src/gpu/kernels/sana_linear_attention_apply.wgsl +43 -0
- package/src/gpu/kernels/sana_linear_attention_apply_f16.wgsl +46 -0
- package/src/gpu/kernels/sana_linear_attention_summary.wgsl +51 -0
- package/src/gpu/kernels/sana_linear_attention_summary_f16.wgsl +53 -0
- package/src/gpu/kernels/scale.d.ts +35 -0
- package/src/gpu/kernels/scale.js +44 -0
- package/src/gpu/kernels/scale.wgsl +38 -0
- package/src/gpu/kernels/scatter_add.wgsl +88 -0
- package/src/gpu/kernels/scatter_add_dynamic.wgsl +59 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +52 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights.wgsl +50 -0
- package/src/gpu/kernels/scatter_add_vec4.wgsl +70 -0
- package/src/gpu/kernels/shader-cache.d.ts +56 -0
- package/src/gpu/kernels/shader-cache.js +234 -0
- package/src/gpu/kernels/silu.d.ts +76 -0
- package/src/gpu/kernels/silu.js +406 -0
- package/src/gpu/kernels/silu.wgsl +109 -0
- package/src/gpu/kernels/silu_f16.wgsl +108 -0
- package/src/gpu/kernels/softmax.d.ts +57 -0
- package/src/gpu/kernels/softmax.js +125 -0
- package/src/gpu/kernels/softmax.wgsl +388 -0
- package/src/gpu/kernels/softmax_subgroup.wgsl +175 -0
- package/src/gpu/kernels/split_qg.d.ts +50 -0
- package/src/gpu/kernels/split_qg.js +46 -0
- package/src/gpu/kernels/split_qg.wgsl +58 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +62 -0
- package/src/gpu/kernels/split_qkv.d.ts +51 -0
- package/src/gpu/kernels/split_qkv.js +51 -0
- package/src/gpu/kernels/split_qkv.wgsl +71 -0
- package/src/gpu/kernels/split_qkv_f16.wgsl +75 -0
- package/src/gpu/kernels/topk.wgsl +243 -0
- package/src/gpu/kernels/topk_f16.wgsl +108 -0
- package/src/gpu/kernels/topk_f16_weights.wgsl +101 -0
- package/src/gpu/kernels/transpose.d.ts +21 -0
- package/src/gpu/kernels/transpose.js +51 -0
- package/src/gpu/kernels/transpose.wgsl +33 -0
- package/src/gpu/kernels/types.d.ts +21 -0
- package/src/gpu/kernels/types.js +4 -0
- package/src/gpu/kernels/uniform-utils.d.ts +48 -0
- package/src/gpu/kernels/uniform-utils.js +94 -0
- package/src/gpu/kernels/upsample2d.d.ts +25 -0
- package/src/gpu/kernels/upsample2d.js +67 -0
- package/src/gpu/kernels/upsample2d.wgsl +34 -0
- package/src/gpu/kernels/upsample2d_f16.wgsl +38 -0
- package/src/gpu/kernels/utils.d.ts +106 -0
- package/src/gpu/kernels/utils.js +257 -0
- package/src/gpu/multi-model-recorder.d.ts +21 -0
- package/src/gpu/multi-model-recorder.js +31 -0
- package/src/gpu/partitioned-buffer-pool.d.ts +28 -0
- package/src/gpu/partitioned-buffer-pool.js +57 -0
- package/src/gpu/perf-guards.d.ts +25 -0
- package/src/gpu/perf-guards.js +133 -0
- package/src/gpu/profiler.d.ts +114 -0
- package/src/gpu/profiler.js +406 -0
- package/src/gpu/readback-utils.d.ts +16 -0
- package/src/gpu/readback-utils.js +41 -0
- package/src/gpu/submit-tracker.d.ts +111 -0
- package/src/gpu/submit-tracker.js +242 -0
- package/src/gpu/tensor.d.ts +69 -0
- package/src/gpu/tensor.js +75 -0
- package/src/gpu/uniform-cache.d.ts +109 -0
- package/src/gpu/uniform-cache.js +263 -0
- package/src/gpu/weight-buffer.d.ts +121 -0
- package/src/gpu/weight-buffer.js +132 -0
- package/src/hotswap/intent-bundle.d.ts +37 -0
- package/src/hotswap/intent-bundle.js +129 -0
- package/src/hotswap/manifest.d.ts +42 -0
- package/src/hotswap/manifest.js +124 -0
- package/src/hotswap/model-swap.js +42 -0
- package/src/hotswap/runtime.d.ts +31 -0
- package/src/hotswap/runtime.js +150 -0
- package/src/index-browser.d.ts +92 -0
- package/src/index-browser.js +68 -0
- package/src/index-internal.d.ts +2 -0
- package/src/index-internal.js +2 -0
- package/src/index.d.ts +103 -0
- package/src/index.js +76 -0
- package/src/inference/README.md +593 -0
- package/src/inference/browser-harness-contract-helpers.d.ts +5 -0
- package/src/inference/browser-harness-contract-helpers.js +26 -0
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +2 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -0
- package/src/inference/browser-harness-model-helpers.d.ts +16 -0
- package/src/inference/browser-harness-model-helpers.js +217 -0
- package/src/inference/browser-harness-report-helpers.d.ts +7 -0
- package/src/inference/browser-harness-report-helpers.js +42 -0
- package/src/inference/browser-harness-runtime-helpers.d.ts +61 -0
- package/src/inference/browser-harness-runtime-helpers.js +416 -0
- package/src/inference/browser-harness-suite-helpers.d.ts +28 -0
- package/src/inference/browser-harness-suite-helpers.js +270 -0
- package/src/inference/browser-harness-text-helpers.d.ts +34 -0
- package/src/inference/browser-harness-text-helpers.js +879 -0
- package/src/inference/browser-harness.d.ts +249 -0
- package/src/inference/browser-harness.js +1150 -0
- package/src/inference/decode-buffers.d.ts +108 -0
- package/src/inference/decode-buffers.js +181 -0
- package/src/inference/decode-ring.d.ts +52 -0
- package/src/inference/decode-ring.js +273 -0
- package/src/inference/expert-router.d.ts +27 -0
- package/src/inference/expert-router.js +55 -0
- package/src/inference/fixtures/embedding-semantic-fixtures.json +382 -0
- package/src/inference/functiongemma.d.ts +15 -0
- package/src/inference/functiongemma.js +1 -0
- package/src/inference/kv-cache/base.d.ts +150 -0
- package/src/inference/kv-cache/base.js +1105 -0
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +50 -0
- package/src/inference/kv-cache/basis-decomposed-paged.js +276 -0
- package/src/inference/kv-cache/index.d.ts +35 -0
- package/src/inference/kv-cache/index.js +20 -0
- package/src/inference/kv-cache/sliding-window.d.ts +72 -0
- package/src/inference/kv-cache/sliding-window.js +243 -0
- package/src/inference/kv-cache/tiered.d.ts +89 -0
- package/src/inference/kv-cache/tiered.js +576 -0
- package/src/inference/kv-cache/types.d.ts +188 -0
- package/src/inference/kv-cache/types.js +80 -0
- package/src/inference/kv-cache.d.ts +36 -0
- package/src/inference/kv-cache.js +18 -0
- package/src/inference/moe-router.d.ts +212 -0
- package/src/inference/moe-router.js +600 -0
- package/src/inference/multi-model-network.d.ts +139 -0
- package/src/inference/multi-model-network.js +771 -0
- package/src/inference/multi-pipeline-pool.d.ts +62 -0
- package/src/inference/multi-pipeline-pool.js +161 -0
- package/src/inference/network-evolution.d.ts +55 -0
- package/src/inference/network-evolution.js +79 -0
- package/src/inference/pipelines/context.d.ts +21 -0
- package/src/inference/pipelines/context.js +184 -0
- package/src/inference/pipelines/diffusion/helpers.d.ts +29 -0
- package/src/inference/pipelines/diffusion/helpers.js +120 -0
- package/src/inference/pipelines/diffusion/index.d.ts +3 -0
- package/src/inference/pipelines/diffusion/index.js +3 -0
- package/src/inference/pipelines/diffusion/init.d.ts +24 -0
- package/src/inference/pipelines/diffusion/init.js +138 -0
- package/src/inference/pipelines/diffusion/pipeline.d.ts +38 -0
- package/src/inference/pipelines/diffusion/pipeline.js +782 -0
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +53 -0
- package/src/inference/pipelines/diffusion/sana-transformer.js +738 -0
- package/src/inference/pipelines/diffusion/scheduler.d.ts +35 -0
- package/src/inference/pipelines/diffusion/scheduler.js +153 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +20 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +1194 -0
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +21 -0
- package/src/inference/pipelines/diffusion/sd3-weights.js +287 -0
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +87 -0
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +1224 -0
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +29 -0
- package/src/inference/pipelines/diffusion/text-encoder.js +195 -0
- package/src/inference/pipelines/diffusion/types.d.ts +116 -0
- package/src/inference/pipelines/diffusion/types.js +1 -0
- package/src/inference/pipelines/diffusion/vae.d.ts +20 -0
- package/src/inference/pipelines/diffusion/vae.js +1375 -0
- package/src/inference/pipelines/diffusion/weights.d.ts +40 -0
- package/src/inference/pipelines/diffusion/weights.js +155 -0
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +29 -0
- package/src/inference/pipelines/dream/energy-head-pipeline.js +6 -0
- package/src/inference/pipelines/dream/pipeline.d.ts +17 -0
- package/src/inference/pipelines/dream/pipeline.js +8 -0
- package/src/inference/pipelines/energy/index.d.ts +1 -0
- package/src/inference/pipelines/energy/index.js +1 -0
- package/src/inference/pipelines/energy/pipeline.d.ts +27 -0
- package/src/inference/pipelines/energy/pipeline.js +697 -0
- package/src/inference/pipelines/energy/quintel.d.ts +92 -0
- package/src/inference/pipelines/energy/quintel.js +218 -0
- package/src/inference/pipelines/energy/types.d.ts +63 -0
- package/src/inference/pipelines/energy/types.js +1 -0
- package/src/inference/pipelines/energy-head/index.d.ts +6 -0
- package/src/inference/pipelines/energy-head/index.js +6 -0
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +103 -0
- package/src/inference/pipelines/energy-head/row-head-pipeline.js +491 -0
- package/src/inference/pipelines/factory.d.ts +10 -0
- package/src/inference/pipelines/factory.js +6 -0
- package/src/inference/pipelines/index.d.ts +22 -0
- package/src/inference/pipelines/index.js +19 -0
- package/src/inference/pipelines/registry.d.ts +15 -0
- package/src/inference/pipelines/registry.js +23 -0
- package/src/inference/pipelines/rng.d.ts +2 -0
- package/src/inference/pipelines/rng.js +17 -0
- package/src/inference/pipelines/structured/index.d.ts +8 -0
- package/src/inference/pipelines/structured/index.js +8 -0
- package/src/inference/pipelines/structured/json-head-pipeline.d.ts +58 -0
- package/src/inference/pipelines/structured/json-head-pipeline.js +196 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +28 -0
- package/src/inference/pipelines/text/attention/attn-config.js +51 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +148 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +251 -0
- package/src/inference/pipelines/text/attention/index.d.ts +24 -0
- package/src/inference/pipelines/text/attention/index.js +17 -0
- package/src/inference/pipelines/text/attention/output-projection.d.ts +12 -0
- package/src/inference/pipelines/text/attention/output-projection.js +8 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +114 -0
- package/src/inference/pipelines/text/attention/projections.js +531 -0
- package/src/inference/pipelines/text/attention/record.d.ts +36 -0
- package/src/inference/pipelines/text/attention/record.js +523 -0
- package/src/inference/pipelines/text/attention/run.d.ts +38 -0
- package/src/inference/pipelines/text/attention/run.js +813 -0
- package/src/inference/pipelines/text/attention/types.d.ts +108 -0
- package/src/inference/pipelines/text/attention/types.js +67 -0
- package/src/inference/pipelines/text/attention.d.ts +23 -0
- package/src/inference/pipelines/text/attention.js +12 -0
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +22 -0
- package/src/inference/pipelines/text/bdpa-steamroller.js +158 -0
- package/src/inference/pipelines/text/buffer-types.d.ts +7 -0
- package/src/inference/pipelines/text/buffer-types.js +4 -0
- package/src/inference/pipelines/text/chat-format.d.ts +46 -0
- package/src/inference/pipelines/text/chat-format.js +365 -0
- package/src/inference/pipelines/text/config.d.ts +253 -0
- package/src/inference/pipelines/text/config.js +857 -0
- package/src/inference/pipelines/text/debug-utils/config.d.ts +144 -0
- package/src/inference/pipelines/text/debug-utils/config.js +156 -0
- package/src/inference/pipelines/text/debug-utils/index.d.ts +53 -0
- package/src/inference/pipelines/text/debug-utils/index.js +44 -0
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +106 -0
- package/src/inference/pipelines/text/debug-utils/logging.js +152 -0
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +119 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +268 -0
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +77 -0
- package/src/inference/pipelines/text/debug-utils/utils.js +139 -0
- package/src/inference/pipelines/text/debug-utils.d.ts +42 -0
- package/src/inference/pipelines/text/debug-utils.js +34 -0
- package/src/inference/pipelines/text/drift-policy.d.ts +57 -0
- package/src/inference/pipelines/text/drift-policy.js +133 -0
- package/src/inference/pipelines/text/dtype-contract.d.ts +38 -0
- package/src/inference/pipelines/text/dtype-contract.js +87 -0
- package/src/inference/pipelines/text/embed.d.ts +67 -0
- package/src/inference/pipelines/text/embed.js +490 -0
- package/src/inference/pipelines/text/execution-plan.d.ts +123 -0
- package/src/inference/pipelines/text/execution-plan.js +371 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +45 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +382 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +45 -0
- package/src/inference/pipelines/text/execution-v1.js +288 -0
- package/src/inference/pipelines/text/failure-reduction.d.ts +84 -0
- package/src/inference/pipelines/text/failure-reduction.js +180 -0
- package/src/inference/pipelines/text/ffn/dense.d.ts +40 -0
- package/src/inference/pipelines/text/ffn/dense.js +753 -0
- package/src/inference/pipelines/text/ffn/index.d.ts +23 -0
- package/src/inference/pipelines/text/ffn/index.js +16 -0
- package/src/inference/pipelines/text/ffn/moe.d.ts +21 -0
- package/src/inference/pipelines/text/ffn/moe.js +49 -0
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +25 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +196 -0
- package/src/inference/pipelines/text/ffn/standard.d.ts +23 -0
- package/src/inference/pipelines/text/ffn/standard.js +113 -0
- package/src/inference/pipelines/text/ffn/types.d.ts +30 -0
- package/src/inference/pipelines/text/ffn/types.js +25 -0
- package/src/inference/pipelines/text/ffn.d.ts +31 -0
- package/src/inference/pipelines/text/ffn.js +18 -0
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +11 -0
- package/src/inference/pipelines/text/finiteness-guard-status.js +21 -0
- package/src/inference/pipelines/text/finiteness-policy.d.ts +35 -0
- package/src/inference/pipelines/text/finiteness-policy.js +45 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +34 -0
- package/src/inference/pipelines/text/generator-helpers.js +178 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +95 -0
- package/src/inference/pipelines/text/generator-runtime.js +492 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +136 -0
- package/src/inference/pipelines/text/generator-steps.js +1294 -0
- package/src/inference/pipelines/text/generator.d.ts +46 -0
- package/src/inference/pipelines/text/generator.js +1779 -0
- package/src/inference/pipelines/text/index.d.ts +5 -0
- package/src/inference/pipelines/text/index.js +6 -0
- package/src/inference/pipelines/text/init.d.ts +316 -0
- package/src/inference/pipelines/text/init.js +1187 -0
- package/src/inference/pipelines/text/kernel-trace.d.ts +152 -0
- package/src/inference/pipelines/text/kernel-trace.js +330 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +65 -0
- package/src/inference/pipelines/text/layer-plan.js +249 -0
- package/src/inference/pipelines/text/layer.d.ts +56 -0
- package/src/inference/pipelines/text/layer.js +974 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +109 -0
- package/src/inference/pipelines/text/linear-attention.js +948 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +81 -0
- package/src/inference/pipelines/text/logits/cpu.js +91 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +113 -0
- package/src/inference/pipelines/text/logits/gpu.js +419 -0
- package/src/inference/pipelines/text/logits/index.d.ts +62 -0
- package/src/inference/pipelines/text/logits/index.js +316 -0
- package/src/inference/pipelines/text/logits/types.d.ts +46 -0
- package/src/inference/pipelines/text/logits/types.js +4 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +56 -0
- package/src/inference/pipelines/text/logits/utils.js +68 -0
- package/src/inference/pipelines/text/logits.d.ts +27 -0
- package/src/inference/pipelines/text/logits.js +16 -0
- package/src/inference/pipelines/text/lora-apply.d.ts +28 -0
- package/src/inference/pipelines/text/lora-apply.js +76 -0
- package/src/inference/pipelines/text/lora-types.d.ts +39 -0
- package/src/inference/pipelines/text/lora-types.js +18 -0
- package/src/inference/pipelines/text/lora.d.ts +18 -0
- package/src/inference/pipelines/text/lora.js +12 -0
- package/src/inference/pipelines/text/model-load.d.ts +58 -0
- package/src/inference/pipelines/text/model-load.js +778 -0
- package/src/inference/pipelines/text/moe-cache.d.ts +32 -0
- package/src/inference/pipelines/text/moe-cache.js +108 -0
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +9 -0
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +115 -0
- package/src/inference/pipelines/text/moe-cpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-cpu.js +120 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +653 -0
- package/src/inference/pipelines/text/moe-helpers.d.ts +12 -0
- package/src/inference/pipelines/text/moe-helpers.js +21 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +117 -0
- package/src/inference/pipelines/text/moe-impl.js +9 -0
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +40 -0
- package/src/inference/pipelines/text/moe-shape-validator.js +98 -0
- package/src/inference/pipelines/text/operator-events.d.ts +125 -0
- package/src/inference/pipelines/text/operator-events.js +305 -0
- package/src/inference/pipelines/text/operator-identity.d.ts +60 -0
- package/src/inference/pipelines/text/operator-identity.js +133 -0
- package/src/inference/pipelines/text/ops.d.ts +167 -0
- package/src/inference/pipelines/text/ops.js +437 -0
- package/src/inference/pipelines/text/probes.d.ts +31 -0
- package/src/inference/pipelines/text/probes.js +240 -0
- package/src/inference/pipelines/text/sampling-config.d.ts +54 -0
- package/src/inference/pipelines/text/sampling-config.js +71 -0
- package/src/inference/pipelines/text/sampling.d.ts +54 -0
- package/src/inference/pipelines/text/sampling.js +249 -0
- package/src/inference/pipelines/text/stage-names.d.ts +88 -0
- package/src/inference/pipelines/text/stage-names.js +202 -0
- package/src/inference/pipelines/text/state.d.ts +114 -0
- package/src/inference/pipelines/text/state.js +165 -0
- package/src/inference/pipelines/text/types.d.ts +682 -0
- package/src/inference/pipelines/text/types.js +4 -0
- package/src/inference/pipelines/text/weights.d.ts +110 -0
- package/src/inference/pipelines/text/weights.js +173 -0
- package/src/inference/pipelines/text.d.ts +162 -0
- package/src/inference/pipelines/text.js +852 -0
- package/src/inference/pipelines/vision/encoder.js +386 -0
- package/src/inference/pipelines/vision/image-preprocess.js +157 -0
- package/src/inference/pipelines/vision/index.js +173 -0
- package/src/inference/pipelines/vision/ops.js +78 -0
- package/src/inference/pipelines/vision/patch-embed.js +151 -0
- package/src/inference/speculative.d.ts +239 -0
- package/src/inference/speculative.js +402 -0
- package/src/inference/test-harness.d.ts +178 -0
- package/src/inference/test-harness.js +361 -0
- package/src/inference/tokenizer.d.ts +72 -0
- package/src/inference/tokenizer.js +251 -0
- package/src/inference/tokenizers/base.d.ts +39 -0
- package/src/inference/tokenizers/base.js +69 -0
- package/src/inference/tokenizers/bpe.d.ts +27 -0
- package/src/inference/tokenizers/bpe.js +180 -0
- package/src/inference/tokenizers/bundled.d.ts +63 -0
- package/src/inference/tokenizers/bundled.js +1009 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +28 -0
- package/src/inference/tokenizers/sentencepiece.js +401 -0
- package/src/inference/tokenizers/types.d.ts +166 -0
- package/src/inference/tokenizers/types.js +7 -0
- package/src/loader/doppler-loader.d.ts +139 -0
- package/src/loader/doppler-loader.js +1103 -0
- package/src/loader/dtype-utils.d.ts +40 -0
- package/src/loader/dtype-utils.js +61 -0
- package/src/loader/embedding-loader.d.ts +56 -0
- package/src/loader/embedding-loader.js +211 -0
- package/src/loader/experts/expert-cache.d.ts +156 -0
- package/src/loader/experts/expert-cache.js +386 -0
- package/src/loader/experts/expert-loader.d.ts +108 -0
- package/src/loader/experts/expert-loader.js +392 -0
- package/src/loader/final-weights-loader.d.ts +88 -0
- package/src/loader/final-weights-loader.js +339 -0
- package/src/loader/index.d.ts +150 -0
- package/src/loader/index.js +124 -0
- package/src/loader/layer-loader.d.ts +63 -0
- package/src/loader/layer-loader.js +474 -0
- package/src/loader/loader-state.d.ts +51 -0
- package/src/loader/loader-state.js +142 -0
- package/src/loader/loader-types.d.ts +236 -0
- package/src/loader/loader-types.js +4 -0
- package/src/loader/manifest-config.d.ts +97 -0
- package/src/loader/manifest-config.js +134 -0
- package/src/loader/memory-monitor.d.ts +112 -0
- package/src/loader/memory-monitor.js +284 -0
- package/src/loader/multi-model-loader.d.ts +51 -0
- package/src/loader/multi-model-loader.js +133 -0
- package/src/loader/quantization-constants.d.ts +23 -0
- package/src/loader/quantization-constants.js +14 -0
- package/src/loader/shard-cache.d.ts +60 -0
- package/src/loader/shard-cache.js +663 -0
- package/src/loader/shard-resolver.d.ts +12 -0
- package/src/loader/shard-resolver.js +144 -0
- package/src/loader/tensors/tensor-loader.d.ts +157 -0
- package/src/loader/tensors/tensor-loader.js +652 -0
- package/src/loader/tensors/tensor-reader.d.ts +22 -0
- package/src/loader/tensors/tensor-reader.js +113 -0
- package/src/loader/tensors/tensor-role.d.ts +7 -0
- package/src/loader/tensors/tensor-role.js +12 -0
- package/src/loader/weight-downcast.d.ts +62 -0
- package/src/loader/weight-downcast.js +213 -0
- package/src/loader/weights.d.ts +22 -0
- package/src/loader/weights.js +4 -0
- package/src/memory/address-table.d.ts +104 -0
- package/src/memory/address-table.js +114 -0
- package/src/memory/buffer-pool.d.ts +204 -0
- package/src/memory/buffer-pool.js +867 -0
- package/src/memory/capability.d.ts +49 -0
- package/src/memory/capability.js +95 -0
- package/src/memory/heap-manager.d.ts +104 -0
- package/src/memory/heap-manager.js +264 -0
- package/src/memory/unified-detect.d.ts +59 -0
- package/src/memory/unified-detect.js +192 -0
- package/src/rules/converter/execution.rules.json +20 -0
- package/src/rules/converter/tensor-roles.rules.json +13 -0
- package/src/rules/converter/tokenizer.rules.json +7 -0
- package/src/rules/execution-rules-contract-check.d.ts +17 -0
- package/src/rules/execution-rules-contract-check.js +287 -0
- package/src/rules/inference/attention.rules.json +54 -0
- package/src/rules/inference/capability-transforms.rules.json +24 -0
- package/src/rules/inference/config.rules.json +57 -0
- package/src/rules/inference/dtype.rules.json +99 -0
- package/src/rules/inference/execution.rules.json +56 -0
- package/src/rules/inference/ffn.rules.json +35 -0
- package/src/rules/inference/layer-pattern.rules.json +16 -0
- package/src/rules/inference/layer.rules.json +7 -0
- package/src/rules/inference/moe.rules.json +48 -0
- package/src/rules/kernels/attention.rules.json +61 -0
- package/src/rules/kernels/conv2d.rules.json +6 -0
- package/src/rules/kernels/depthwise-conv2d.rules.json +6 -0
- package/src/rules/kernels/dequant.rules.json +58 -0
- package/src/rules/kernels/energy.rules.json +22 -0
- package/src/rules/kernels/fused-ffn.rules.json +13 -0
- package/src/rules/kernels/fused-matmul-residual.rules.json +6 -0
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +8 -0
- package/src/rules/kernels/gather.rules.json +12 -0
- package/src/rules/kernels/gelu.rules.json +11 -0
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +6 -0
- package/src/rules/kernels/groupnorm.rules.json +10 -0
- package/src/rules/kernels/kernel-validator.d.ts +24 -0
- package/src/rules/kernels/kernel-validator.js +160 -0
- package/src/rules/kernels/kv_quantize.rules.json +7 -0
- package/src/rules/kernels/layernorm.rules.json +6 -0
- package/src/rules/kernels/matmul.rules.json +60 -0
- package/src/rules/kernels/modulate.rules.json +6 -0
- package/src/rules/kernels/moe.rules.gptoss.json +105 -0
- package/src/rules/kernels/moe.rules.json +11 -0
- package/src/rules/kernels/moe.rules.mixtral.json +75 -0
- package/src/rules/kernels/pixel_shuffle.rules.json +6 -0
- package/src/rules/kernels/relu.rules.json +6 -0
- package/src/rules/kernels/repeat-channels.rules.json +6 -0
- package/src/rules/kernels/residual.rules.json +12 -0
- package/src/rules/kernels/rmsnorm.rules.json +11 -0
- package/src/rules/kernels/rope.rules.json +6 -0
- package/src/rules/kernels/sample.rules.json +6 -0
- package/src/rules/kernels/sana-linear-attention.rules.json +6 -0
- package/src/rules/kernels/scale.rules.json +6 -0
- package/src/rules/kernels/silu.rules.json +21 -0
- package/src/rules/kernels/softmax.rules.json +25 -0
- package/src/rules/kernels/split-qg.rules.json +6 -0
- package/src/rules/kernels/split-qkv.rules.json +6 -0
- package/src/rules/kernels/upsample2d.rules.json +6 -0
- package/src/rules/layer-pattern-contract-check.d.ts +17 -0
- package/src/rules/layer-pattern-contract-check.js +231 -0
- package/src/rules/loader/tensor-loader.rules.json +24 -0
- package/src/rules/loader/weights.rules.json +41 -0
- package/src/rules/rule-registry.d.ts +77 -0
- package/src/rules/rule-registry.js +264 -0
- package/src/rules/tooling/command-runtime.rules.json +65 -0
- package/src/storage/backends/idb-store.d.ts +52 -0
- package/src/storage/backends/idb-store.js +590 -0
- package/src/storage/backends/memory-store.d.ts +36 -0
- package/src/storage/backends/memory-store.js +242 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/storage/backends/opfs-store.js +478 -0
- package/src/storage/blake3.d.ts +17 -0
- package/src/storage/blake3.js +269 -0
- package/src/storage/download-types.d.ts +157 -0
- package/src/storage/download-types.js +48 -0
- package/src/storage/downloader.d.ts +103 -0
- package/src/storage/downloader.js +1121 -0
- package/src/storage/emulated-vram.d.ts +264 -0
- package/src/storage/emulated-vram.js +576 -0
- package/src/storage/export.d.ts +20 -0
- package/src/storage/export.js +159 -0
- package/src/storage/index.d.ts +256 -0
- package/src/storage/index.js +188 -0
- package/src/storage/inventory.d.ts +26 -0
- package/src/storage/inventory.js +218 -0
- package/src/storage/preflight.d.ts +144 -0
- package/src/storage/preflight.js +316 -0
- package/src/storage/quickstart-downloader.d.ts +157 -0
- package/src/storage/quickstart-downloader.js +268 -0
- package/src/storage/quota.d.ts +150 -0
- package/src/storage/quota.js +305 -0
- package/src/storage/registry.d.ts +28 -0
- package/src/storage/registry.js +131 -0
- package/src/storage/reports.d.ts +20 -0
- package/src/storage/reports.js +94 -0
- package/src/storage/shard-manager.d.ts +151 -0
- package/src/storage/shard-manager.js +861 -0
- package/src/storage/source-artifact-store.d.ts +52 -0
- package/src/storage/source-artifact-store.js +234 -0
- package/src/sw.d.ts +1 -0
- package/src/sw.js +187 -0
- package/src/tooling/bench-runner.js +64 -0
- package/src/tooling/browser-command-runner.d.ts +28 -0
- package/src/tooling/browser-command-runner.js +85 -0
- package/src/tooling/command-api-constants.d.ts +9 -0
- package/src/tooling/command-api-constants.js +9 -0
- package/src/tooling/command-api-family-normalizers.d.ts +9 -0
- package/src/tooling/command-api-family-normalizers.js +395 -0
- package/src/tooling/command-api-helpers.d.ts +25 -0
- package/src/tooling/command-api-helpers.js +265 -0
- package/src/tooling/command-api.d.ts +177 -0
- package/src/tooling/command-api.js +82 -0
- package/src/tooling/command-envelope.d.ts +84 -0
- package/src/tooling/command-envelope.js +201 -0
- package/src/tooling/command-runner-shared.d.ts +75 -0
- package/src/tooling/command-runner-shared.js +191 -0
- package/src/tooling/command-runner.html +45 -0
- package/src/tooling/command-validation.js +44 -0
- package/src/tooling/conversion-config-materializer.d.ts +24 -0
- package/src/tooling/conversion-config-materializer.js +96 -0
- package/src/tooling/diagnose-runner.d.ts +28 -0
- package/src/tooling/diagnose-runner.js +333 -0
- package/src/tooling/distill-runner.js +56 -0
- package/src/tooling/hf-registry-utils.d.ts +59 -0
- package/src/tooling/hf-registry-utils.js +324 -0
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +18 -0
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +72 -0
- package/src/tooling/kernel-path-builder/index.d.ts +65 -0
- package/src/tooling/kernel-path-builder/index.js +1348 -0
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +9 -0
- package/src/tooling/kernel-path-builder/runtime-overlay.js +149 -0
- package/src/tooling/lean-execution-contract-runner.d.ts +43 -0
- package/src/tooling/lean-execution-contract-runner.js +158 -0
- package/src/tooling/lean-execution-contract.d.ts +16 -0
- package/src/tooling/lean-execution-contract.js +228 -0
- package/src/tooling/lora-runner.js +37 -0
- package/src/tooling/node-browser-command-runner.d.ts +34 -0
- package/src/tooling/node-browser-command-runner.js +827 -0
- package/src/tooling/node-command-runner.d.ts +36 -0
- package/src/tooling/node-command-runner.js +187 -0
- package/src/tooling/node-convert-worker-pool.d.ts +16 -0
- package/src/tooling/node-convert-worker-pool.js +186 -0
- package/src/tooling/node-convert-worker.d.ts +1 -0
- package/src/tooling/node-convert-worker.js +60 -0
- package/src/tooling/node-converter.d.ts +1 -0
- package/src/tooling/node-converter.js +1330 -0
- package/src/tooling/node-file-fetch.d.ts +1 -0
- package/src/tooling/node-file-fetch.js +38 -0
- package/src/tooling/node-source-runtime.d.ts +19 -0
- package/src/tooling/node-source-runtime.js +612 -0
- package/src/tooling/node-webgpu.d.ts +16 -0
- package/src/tooling/node-webgpu.js +421 -0
- package/src/tooling/opfs-cache.d.ts +12 -0
- package/src/tooling/opfs-cache.js +260 -0
- package/src/tooling/runtime-input-composition.d.ts +38 -0
- package/src/tooling/runtime-input-composition.js +90 -0
- package/src/tooling/source-runtime-bundle.d.ts +141 -0
- package/src/tooling/source-runtime-bundle.js +712 -0
- package/src/tooling/source-runtime-materializer.d.ts +6 -0
- package/src/tooling/source-runtime-materializer.js +93 -0
- package/src/tooling/verify-runner.js +63 -0
- package/src/tooling-exports.browser.d.ts +7 -0
- package/src/tooling-exports.browser.js +2 -0
- package/src/tooling-exports.d.ts +22 -0
- package/src/tooling-exports.js +7 -0
- package/src/tooling-exports.shared.d.ts +106 -0
- package/src/tooling-exports.shared.js +93 -0
- package/src/training/README.md +153 -0
- package/src/training/artifacts.d.ts +160 -0
- package/src/training/artifacts.js +896 -0
- package/src/training/attention-backward.d.ts +30 -0
- package/src/training/attention-backward.js +232 -0
- package/src/training/attention-forward.d.ts +22 -0
- package/src/training/attention-forward.js +82 -0
- package/src/training/autograd.d.ts +51 -0
- package/src/training/autograd.js +408 -0
- package/src/training/backward-kernels.js +39 -0
- package/src/training/checkpoint-watch.d.ts +8 -0
- package/src/training/checkpoint-watch.js +139 -0
- package/src/training/checkpoint.d.ts +36 -0
- package/src/training/checkpoint.js +281 -0
- package/src/training/clip.d.ts +9 -0
- package/src/training/clip.js +55 -0
- package/src/training/dataloader.d.ts +8 -0
- package/src/training/dataloader.js +44 -0
- package/src/training/datasets/index.d.ts +12 -0
- package/src/training/datasets/index.js +6 -0
- package/src/training/datasets/jsonl.d.ts +11 -0
- package/src/training/datasets/jsonl.js +50 -0
- package/src/training/datasets/reploid.d.ts +3 -0
- package/src/training/datasets/reploid.js +36 -0
- package/src/training/datasets/text-pairs.d.ts +21 -0
- package/src/training/datasets/text-pairs.js +42 -0
- package/src/training/datasets/token-batch.d.ts +21 -0
- package/src/training/datasets/token-batch.js +52 -0
- package/src/training/datasets/translation-pairs.d.ts +34 -0
- package/src/training/datasets/translation-pairs.js +49 -0
- package/src/training/distillation/artifacts.d.ts +71 -0
- package/src/training/distillation/artifacts.js +132 -0
- package/src/training/distillation/checkpoint-watch.d.ts +10 -0
- package/src/training/distillation/checkpoint-watch.js +58 -0
- package/src/training/distillation/dataset.d.ts +59 -0
- package/src/training/distillation/dataset.js +337 -0
- package/src/training/distillation/eval.d.ts +34 -0
- package/src/training/distillation/eval.js +313 -0
- package/src/training/distillation/index.d.ts +29 -0
- package/src/training/distillation/index.js +29 -0
- package/src/training/distillation/runtime.d.ts +20 -0
- package/src/training/distillation/runtime.js +121 -0
- package/src/training/distillation/scoreboard.d.ts +6 -0
- package/src/training/distillation/scoreboard.js +8 -0
- package/src/training/distillation/stage-a.d.ts +45 -0
- package/src/training/distillation/stage-a.js +338 -0
- package/src/training/distillation/stage-b.d.ts +24 -0
- package/src/training/distillation/stage-b.js +20 -0
- package/src/training/distillation/student-fixture.d.ts +22 -0
- package/src/training/distillation/student-fixture.js +846 -0
- package/src/training/distillation/suite-data.d.ts +45 -0
- package/src/training/distillation/suite-data.js +189 -0
- package/src/training/export.d.ts +32 -0
- package/src/training/export.js +112 -0
- package/src/training/index.d.ts +62 -0
- package/src/training/index.js +51 -0
- package/src/training/lora-pipeline.d.ts +40 -0
- package/src/training/lora-pipeline.js +793 -0
- package/src/training/lora.d.ts +19 -0
- package/src/training/lora.js +71 -0
- package/src/training/loss-scaling.d.ts +21 -0
- package/src/training/loss-scaling.js +80 -0
- package/src/training/loss.d.ts +10 -0
- package/src/training/loss.js +40 -0
- package/src/training/objectives/base.d.ts +58 -0
- package/src/training/objectives/base.js +38 -0
- package/src/training/objectives/cross_entropy.d.ts +18 -0
- package/src/training/objectives/cross_entropy.js +34 -0
- package/src/training/objectives/distill_kd.d.ts +16 -0
- package/src/training/objectives/distill_kd.js +369 -0
- package/src/training/objectives/distill_triplet.d.ts +16 -0
- package/src/training/objectives/distill_triplet.js +414 -0
- package/src/training/objectives/index.d.ts +12 -0
- package/src/training/objectives/index.js +6 -0
- package/src/training/objectives/ul_stage1_joint.d.ts +16 -0
- package/src/training/objectives/ul_stage1_joint.js +188 -0
- package/src/training/objectives/ul_stage2_base.d.ts +16 -0
- package/src/training/objectives/ul_stage2_base.js +218 -0
- package/src/training/operator-artifacts.d.ts +62 -0
- package/src/training/operator-artifacts.js +140 -0
- package/src/training/operator-command.d.ts +5 -0
- package/src/training/operator-command.js +455 -0
- package/src/training/operator-eval.d.ts +48 -0
- package/src/training/operator-eval.js +230 -0
- package/src/training/operator-scoreboard.d.ts +5 -0
- package/src/training/operator-scoreboard.js +44 -0
- package/src/training/optimizer.d.ts +22 -0
- package/src/training/optimizer.js +127 -0
- package/src/training/runner.d.ts +248 -0
- package/src/training/runner.js +1236 -0
- package/src/training/suite.d.ts +299 -0
- package/src/training/suite.js +2196 -0
- package/src/training/tensor-factory.d.ts +9 -0
- package/src/training/tensor-factory.js +13 -0
- package/src/training/trainer.d.ts +89 -0
- package/src/training/trainer.js +299 -0
- package/src/training/ul_dataset.d.ts +47 -0
- package/src/training/ul_dataset.js +151 -0
- package/src/training/ul_schedule.d.ts +6 -0
- package/src/training/ul_schedule.js +29 -0
- package/src/training/workload-packs/distill-translategemma-medium.json +103 -0
- package/src/training/workload-packs/distill-translategemma-tiny.json +103 -0
- package/src/training/workload-packs/lora-toy-medium.json +91 -0
- package/src/training/workload-packs/lora-toy-tiny.json +91 -0
- package/src/training/workload-packs/registry.json +98 -0
- package/src/training/workload-packs/ul-training-medium.json +55 -0
- package/src/training/workload-packs/ul-training-tiny.json +55 -0
- package/src/training/workloads.d.ts +164 -0
- package/src/training/workloads.js +530 -0
- package/src/types/chrome.d.ts +36 -0
- package/src/types/chrome.js +1 -0
- package/src/types/gpu.d.ts +185 -0
- package/src/types/gpu.js +5 -0
- package/src/types/index.d.ts +4 -0
- package/src/types/index.js +4 -0
- package/src/types/inference.d.ts +197 -0
- package/src/types/inference.js +5 -0
- package/src/types/model.d.ts +130 -0
- package/src/types/model.js +5 -0
- package/src/utils/format-bytes.js +20 -0
- package/src/utils/hf-resolve-url.d.ts +16 -0
- package/src/utils/hf-resolve-url.js +17 -0
- package/src/utils/index.d.ts +7 -0
- package/src/utils/index.js +7 -0
- package/src/utils/load-json.d.ts +5 -0
- package/src/utils/load-json.js +33 -0
- package/src/utils/plain-object.d.ts +1 -0
- package/src/utils/plain-object.js +3 -0
- package/src/utils/sha256.d.ts +4 -0
- package/src/utils/sha256.js +135 -0
- package/src/version.d.ts +2 -0
- package/src/version.js +2 -0
- package/tools/convert-safetensors-node.js +240 -0
|
@@ -0,0 +1,1779 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
import { getDevice, setTrackSubmits } from '../../../gpu/device.js';
|
|
4
|
+
import { releaseBuffer, readBuffer, readBufferSlice, uploadData } from '../../../memory/buffer-pool.js';
|
|
5
|
+
import { isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
|
|
6
|
+
import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selection-cache.js';
|
|
7
|
+
import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
|
|
8
|
+
import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
9
|
+
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
10
|
+
import { log, trace } from '../../../debug/index.js';
|
|
11
|
+
import {
|
|
12
|
+
CAPTURE_LEVELS,
|
|
13
|
+
createDefaultCaptureConfig,
|
|
14
|
+
validateCaptureConfig,
|
|
15
|
+
} from '../../../debug/index.js';
|
|
16
|
+
import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
17
|
+
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
18
|
+
|
|
19
|
+
// Pipeline sub-modules
|
|
20
|
+
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
21
|
+
import { enforceLogitDrift } from '../../../hotswap/intent-bundle.js';
|
|
22
|
+
import { applyChatTemplate, isStopToken } from './init.js';
|
|
23
|
+
import { formatChatMessages } from './chat-format.js';
|
|
24
|
+
import { embed } from './embed.js';
|
|
25
|
+
import { processLayer } from './layer.js';
|
|
26
|
+
import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
|
|
27
|
+
import { OperatorEventEmitter } from './operator-events.js';
|
|
28
|
+
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
|
|
29
|
+
import {
|
|
30
|
+
decodeStep,
|
|
31
|
+
decodeStepLogits,
|
|
32
|
+
advanceWithToken,
|
|
33
|
+
generateNTokensGPU,
|
|
34
|
+
shouldUseBatchDecode,
|
|
35
|
+
sumProfileTimings,
|
|
36
|
+
FinitenessError,
|
|
37
|
+
advanceWithTokenAndEmbedding as runAdvanceWithTokenAndEmbedding,
|
|
38
|
+
} from './generator-steps.js';
|
|
39
|
+
import { buildLayerContext, debugCheckBuffer as debugCheckBufferHelper, getLogitsConfig, getLogitsWeights } from './generator-helpers.js';
|
|
40
|
+
import {
|
|
41
|
+
assertTokenIdsInRange,
|
|
42
|
+
assertTokenIdInRange,
|
|
43
|
+
resolveStepOptions,
|
|
44
|
+
resolveGenerateOptions,
|
|
45
|
+
resolvePrefillOptions,
|
|
46
|
+
resolvePrefillEmbeddingOptions,
|
|
47
|
+
resolveAdvanceEmbeddingMode,
|
|
48
|
+
getFinalNormWeights,
|
|
49
|
+
extractEmbeddingFromHidden,
|
|
50
|
+
} from './generator-runtime.js';
|
|
51
|
+
|
|
52
|
+
import { resolveSamplingConfig } from './sampling-config.js';
|
|
53
|
+
import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
|
|
54
|
+
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
55
|
+
import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
|
|
56
|
+
import {
|
|
57
|
+
activateFallbackExecutionPlan,
|
|
58
|
+
rebaseExecutionSessionPlan,
|
|
59
|
+
resetActiveExecutionPlan,
|
|
60
|
+
resolveActiveExecutionPlan,
|
|
61
|
+
setActiveExecutionPlan,
|
|
62
|
+
} from './execution-plan.js';
|
|
63
|
+
import {
|
|
64
|
+
cloneLinearAttentionRuntime,
|
|
65
|
+
hasLinearAttentionLayers,
|
|
66
|
+
resetLinearAttentionRuntime,
|
|
67
|
+
restoreLinearAttentionRuntime,
|
|
68
|
+
} from './linear-attention.js';
|
|
69
|
+
|
|
70
|
+
function isStructuredChatRequest(prompt) {
|
|
71
|
+
return prompt != null
|
|
72
|
+
&& typeof prompt === 'object'
|
|
73
|
+
&& !Array.isArray(prompt)
|
|
74
|
+
&& Array.isArray(prompt.messages);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
|
|
78
|
+
if (typeof prompt === 'string') {
|
|
79
|
+
if (useChatTemplate && state.modelConfig.chatTemplateType) {
|
|
80
|
+
if (state.modelConfig.chatTemplateType === 'translategemma') {
|
|
81
|
+
throw new Error(
|
|
82
|
+
`[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
|
|
83
|
+
'Pass { messages: [...] } instead of a plain string prompt.'
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
return applyChatTemplate(prompt, state.modelConfig.chatTemplateType);
|
|
87
|
+
}
|
|
88
|
+
return prompt;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
|
|
92
|
+
throw new Error(
|
|
93
|
+
`[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
|
|
94
|
+
'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
const messages = isStructuredChatRequest(prompt)
|
|
98
|
+
? prompt.messages
|
|
99
|
+
: (Array.isArray(prompt) ? prompt : null);
|
|
100
|
+
if (!messages) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
|
|
106
|
+
return formatChatMessages(messages, templateType);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Resolve display text for a token ID sequence.
|
|
111
|
+
*
|
|
112
|
+
* Preference order:
|
|
113
|
+
* 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
|
|
114
|
+
* 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
|
|
115
|
+
* 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
|
|
116
|
+
* unless primary returned empty string and fallback looks like a special token (<...>),
|
|
117
|
+
* in which case empty string is preserved to keep skip-special behavior deterministic.
|
|
118
|
+
* 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
|
|
119
|
+
* 5. fallbackText — static fallback string (default '?').
|
|
120
|
+
*/
|
|
121
|
+
function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
|
|
122
|
+
const renderPrimary = typeof renderTokenText === 'function'
|
|
123
|
+
? renderTokenText
|
|
124
|
+
: (ids) => tokenizer?.decode?.(ids);
|
|
125
|
+
const renderFallback = typeof renderFallbackTokenText === 'function'
|
|
126
|
+
? renderFallbackTokenText
|
|
127
|
+
: (ids) => tokenizer?.decode?.(ids, false);
|
|
128
|
+
|
|
129
|
+
const primaryText = renderPrimary(tokenIds);
|
|
130
|
+
if (typeof primaryText === 'string' && primaryText.length > 0) {
|
|
131
|
+
return primaryText;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const fallback = renderFallback(tokenIds);
|
|
135
|
+
if (typeof fallback === 'string' && fallback.length > 0) {
|
|
136
|
+
// Keep skip-special behavior deterministic: if primary decoding filtered this
|
|
137
|
+
// token to empty, do not reintroduce obvious special-token text via fallback.
|
|
138
|
+
if (
|
|
139
|
+
primaryText === ''
|
|
140
|
+
&& /^<[^>\n]{1,80}>$/.test(fallback.trim())
|
|
141
|
+
) {
|
|
142
|
+
return '';
|
|
143
|
+
}
|
|
144
|
+
return fallback;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return fallbackText;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function summarizeExecutionPlan(plan) {
|
|
151
|
+
if (!plan) {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
if (typeof plan !== 'object') {
|
|
155
|
+
log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
if (typeof plan.id !== 'string') {
|
|
159
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
|
|
160
|
+
}
|
|
161
|
+
if (typeof plan.activationDtype !== 'string') {
|
|
162
|
+
log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
id: plan.id,
|
|
166
|
+
kernelPathId: plan.kernelPathId ?? null,
|
|
167
|
+
kernelPathSource: plan.kernelPathSource ?? 'none',
|
|
168
|
+
activationDtype: plan.activationDtype,
|
|
169
|
+
readbackInterval: plan.readbackInterval ?? null,
|
|
170
|
+
batchSize: plan.defaultBatchSize,
|
|
171
|
+
stopCheckMode: plan.defaultStopCheckMode,
|
|
172
|
+
ringTokens: plan.ringTokens ?? null,
|
|
173
|
+
ringStop: plan.ringStop ?? null,
|
|
174
|
+
ringStaging: plan.ringStaging ?? null,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function shouldRetryWithFinitenessFallback(error) {
|
|
179
|
+
if (error?.name === 'FinitenessError') {
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
const message = typeof error?.message === 'string'
|
|
183
|
+
? error.message
|
|
184
|
+
: (typeof error === 'string' ? error : '');
|
|
185
|
+
if (!message.startsWith('[Sampling]')) {
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
return message.includes('no finite candidate logits after masking the pad token')
|
|
189
|
+
|| message.includes('Softmax produced no finite candidate probabilities');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export class PipelineGenerator {
|
|
193
|
+
|
|
194
|
+
#state;
|
|
195
|
+
#finitenessFallbackWindow;
|
|
196
|
+
|
|
197
|
+
_assertTokenIdsInRange(tokenIds, context = 'encode') {
|
|
198
|
+
assertTokenIdsInRange(this.#state, tokenIds, context);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
_assertTokenIdInRange(tokenId, context = 'token') {
|
|
202
|
+
assertTokenIdInRange(this.#state, tokenId, context);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
constructor(state) {
|
|
207
|
+
this.#state = state;
|
|
208
|
+
this.#finitenessFallbackWindow = null;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
_resolveDeferredRoundingWindowTokens() {
|
|
212
|
+
const activePlan = resolveActiveExecutionPlan(this.#state);
|
|
213
|
+
return activePlan?.deferredRoundingWindowTokens
|
|
214
|
+
?? resolveDeferredRoundingWindowTokens(this.#state.runtimeConfig?.inference?.compute);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
_getEffectiveActivationDtype() {
|
|
218
|
+
return resolveActiveExecutionPlan(this.#state).activationDtype;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
_hasFinitenessFallbackWindow() {
|
|
222
|
+
return this.#finitenessFallbackWindow !== null;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
_openFinitenessFallbackWindow(opts, reasonLabel, tokenCount) {
|
|
226
|
+
const normalizedCount = Number.isFinite(tokenCount)
|
|
227
|
+
? Math.max(1, Math.floor(tokenCount))
|
|
228
|
+
: 1;
|
|
229
|
+
if (this.#finitenessFallbackWindow) {
|
|
230
|
+
this.#finitenessFallbackWindow.remainingTokens = Math.max(
|
|
231
|
+
this.#finitenessFallbackWindow.remainingTokens,
|
|
232
|
+
normalizedCount
|
|
233
|
+
);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const original = this._beginFinitenessFallback(opts, reasonLabel);
|
|
237
|
+
this.#finitenessFallbackWindow = {
|
|
238
|
+
original,
|
|
239
|
+
remainingTokens: normalizedCount,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
_closeFinitenessFallbackWindow(opts) {
|
|
244
|
+
if (!this.#finitenessFallbackWindow) {
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const original = this.#finitenessFallbackWindow.original;
|
|
248
|
+
this.#finitenessFallbackWindow = null;
|
|
249
|
+
this._endFinitenessFallback(opts, original);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
_consumeFinitenessFallbackToken(opts) {
|
|
253
|
+
if (!this.#finitenessFallbackWindow) {
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
this.#finitenessFallbackWindow.remainingTokens -= 1;
|
|
257
|
+
if (this.#finitenessFallbackWindow.remainingTokens <= 0) {
|
|
258
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
_resolveStepOptions(options = {}) {
|
|
263
|
+
return resolveStepOptions(this.#state, options);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
_resetDecodeRuntimeState() {
|
|
267
|
+
this.#state.stats.decodeMode = null;
|
|
268
|
+
this.#state.stats.batchGuardReason = null;
|
|
269
|
+
this.#state.stats.decodeProfileSteps = [];
|
|
270
|
+
this.#state.stats.ttftMs = 0;
|
|
271
|
+
this.#state.stats.decodeTimeMs = 0;
|
|
272
|
+
this.#state.stats.decodeRecordMs = 0;
|
|
273
|
+
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
274
|
+
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
275
|
+
this.#state.decodeStepCount = 0;
|
|
276
|
+
this.#state.disableRecordedLogits = false;
|
|
277
|
+
this.#state.disableFusedDecode = false;
|
|
278
|
+
this.#state.batchingStats = {
|
|
279
|
+
batchedForwardCalls: 0,
|
|
280
|
+
unbatchedForwardCalls: 0,
|
|
281
|
+
totalBatchedTimeMs: 0,
|
|
282
|
+
totalUnbatchedTimeMs: 0,
|
|
283
|
+
gpuSubmissions: 0,
|
|
284
|
+
};
|
|
285
|
+
resetActiveExecutionPlan(this.#state);
|
|
286
|
+
this.#state.decodeRing?.reset();
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
_getDecodeHelpers(debugCheckBuffer) {
|
|
290
|
+
return {
|
|
291
|
+
buildLayerContext: (recorder, isDecodeMode, debugLayers, executionPlan) =>
|
|
292
|
+
buildLayerContext(this.#state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
|
|
293
|
+
getLogitsWeights: () => getLogitsWeights(this.#state),
|
|
294
|
+
getLogitsConfig: () => getLogitsConfig(this.#state),
|
|
295
|
+
debugCheckBuffer,
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async _getFinalNormWeights() {
|
|
300
|
+
return getFinalNormWeights(this.#state);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
_extractEmbeddingFromHidden(hiddenStates, numTokens, hiddenSize, embeddingMode, finalNormWeights, config) {
|
|
304
|
+
return extractEmbeddingFromHidden(
|
|
305
|
+
hiddenStates,
|
|
306
|
+
numTokens,
|
|
307
|
+
hiddenSize,
|
|
308
|
+
embeddingMode,
|
|
309
|
+
finalNormWeights,
|
|
310
|
+
config,
|
|
311
|
+
this.#state.embeddingPostprocessor
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
_resolvePromptTokenIds(prompt, useChatTemplate, contextLabel) {
|
|
316
|
+
const processedPrompt = resolvePromptInput(this.#state, prompt, useChatTemplate, contextLabel);
|
|
317
|
+
const inputIds = this.#state.tokenizer.encode(processedPrompt);
|
|
318
|
+
this._assertTokenIdsInRange(inputIds, `${contextLabel}.encode`);
|
|
319
|
+
return inputIds;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
323
|
+
const sampledLogits = Float32Array.from(logits);
|
|
324
|
+
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
325
|
+
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
326
|
+
return sample(sampledLogits, {
|
|
327
|
+
temperature: opts.temperature,
|
|
328
|
+
topP: opts.topP,
|
|
329
|
+
topK: opts.topK,
|
|
330
|
+
padTokenId,
|
|
331
|
+
seed: opts.seed,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async _prefillPromptToLogits(prompt, opts, contextLabel) {
|
|
336
|
+
const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, contextLabel);
|
|
337
|
+
if (opts.debug) {
|
|
338
|
+
log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
let logits;
|
|
342
|
+
try {
|
|
343
|
+
logits = await this._prefill(inputIds, opts);
|
|
344
|
+
} catch (error) {
|
|
345
|
+
if (!shouldRetryWithFinitenessFallback(error)) {
|
|
346
|
+
throw error;
|
|
347
|
+
}
|
|
348
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
|
|
349
|
+
logits = await this._retryWithFinitenessFallback(
|
|
350
|
+
opts,
|
|
351
|
+
contextLabel,
|
|
352
|
+
() => this._prefill(inputIds, opts)
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
return { inputIds, logits };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
async _decodeStepToLogits(currentIds, opts) {
|
|
360
|
+
const debugCheckBuffer = this.#state.debug
|
|
361
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
362
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
363
|
+
: undefined;
|
|
364
|
+
return decodeStepLogits(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async _decodeNextTokenViaLogits(currentIds, opts) {
|
|
368
|
+
const stepResult = await this._decodeStepToLogits(currentIds, opts);
|
|
369
|
+
return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
|
|
373
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
374
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
375
|
+
|
|
376
|
+
validateCallTimeOptions(options);
|
|
377
|
+
|
|
378
|
+
this.#state.isGenerating = true;
|
|
379
|
+
this._resetDecodeRuntimeState();
|
|
380
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
381
|
+
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
382
|
+
this.#state.stats.decodeRecordMs = 0;
|
|
383
|
+
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
384
|
+
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
385
|
+
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
386
|
+
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
387
|
+
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
388
|
+
this.#state.stats.decodeMode = null;
|
|
389
|
+
this.#state.stats.batchGuardReason = null;
|
|
390
|
+
this.#state.stats.ttftMs = 0;
|
|
391
|
+
const startTime = performance.now();
|
|
392
|
+
|
|
393
|
+
const opts = resolveGenerateOptions(this.#state, options);
|
|
394
|
+
// Validate and normalize sampling parameters through single source of truth
|
|
395
|
+
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
396
|
+
opts.temperature = samplingConfig.temperature;
|
|
397
|
+
opts.topP = samplingConfig.topP;
|
|
398
|
+
opts.topK = samplingConfig.topK;
|
|
399
|
+
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
400
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
401
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
402
|
+
if (diagnosticsEnabled) {
|
|
403
|
+
const captureConfig = {
|
|
404
|
+
...createDefaultCaptureConfig(),
|
|
405
|
+
enabled: true,
|
|
406
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
407
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
408
|
+
};
|
|
409
|
+
validateCaptureConfig(captureConfig);
|
|
410
|
+
this.#state.operatorDiagnostics = {
|
|
411
|
+
enabled: true,
|
|
412
|
+
captureConfig,
|
|
413
|
+
emitter: new OperatorEventEmitter({
|
|
414
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
415
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
416
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
417
|
+
}),
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
|
|
421
|
+
this.#state.stats.executionPlan = {
|
|
422
|
+
primary: summarizeExecutionPlan(this.#state.executionPlanState?.primaryPlan ?? null),
|
|
423
|
+
fallback: summarizeExecutionPlan(this.#state.executionPlanState?.fallbackPlan ?? null),
|
|
424
|
+
activePlanIdAtStart: activePlan?.id ?? null,
|
|
425
|
+
finalActivePlanId: activePlan?.id ?? null,
|
|
426
|
+
transitions: [],
|
|
427
|
+
};
|
|
428
|
+
this.#state.stats.kernelPathId = activePlan?.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
429
|
+
this.#state.stats.kernelPathSource = activePlan?.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
430
|
+
|
|
431
|
+
if (opts.debug) {
|
|
432
|
+
log.debug('Pipeline', `ChatTemplate: options=${options.useChatTemplate}, final=${opts.useChatTemplate}`);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const emitToken = async function* (generator, tokenId, textDecoder) {
|
|
436
|
+
if (mode === 'token') {
|
|
437
|
+
yield tokenId;
|
|
438
|
+
if (options.onToken) options.onToken(tokenId, '');
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
441
|
+
const tokenText = textDecoder(tokenId);
|
|
442
|
+
yield tokenText;
|
|
443
|
+
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
try {
|
|
447
|
+
const prefillStart = performance.now();
|
|
448
|
+
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
|
|
449
|
+
let prefillLogits = initialPrefillLogits;
|
|
450
|
+
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
451
|
+
this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
|
|
452
|
+
const generatedIds = [...inputIds];
|
|
453
|
+
this.#state.stats.prefillTokens = inputIds.length;
|
|
454
|
+
|
|
455
|
+
if (opts.debug) {
|
|
456
|
+
log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const intentBundleConfig = this.#state.runtimeConfig.shared.intentBundle;
|
|
460
|
+
const intentBundle = intentBundleConfig?.bundle;
|
|
461
|
+
const expectedTopK = intentBundle?.payload?.expectedTopK
|
|
462
|
+
?? intentBundle?.payload?.expected_top_k;
|
|
463
|
+
const maxDriftThreshold = intentBundle?.constraints?.maxDriftThreshold
|
|
464
|
+
?? intentBundle?.constraints?.max_drift_threshold;
|
|
465
|
+
|
|
466
|
+
if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
|
|
467
|
+
const actualTopK = getTopK(
|
|
468
|
+
prefillLogits,
|
|
469
|
+
expectedTopK.length,
|
|
470
|
+
(tokens) => resolveTokenText(this.#state.tokenizer, tokens),
|
|
471
|
+
).map((token) => token.token);
|
|
472
|
+
const driftResult = enforceLogitDrift(expectedTopK, actualTopK, maxDriftThreshold);
|
|
473
|
+
if (!driftResult.ok) {
|
|
474
|
+
throw new Error(`Intent bundle drift check failed: ${driftResult.reason}`);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (opts.debug) {
|
|
479
|
+
const topAfterPenalty = getTopK(
|
|
480
|
+
Float32Array.from(prefillLogits),
|
|
481
|
+
5,
|
|
482
|
+
(tokens) => resolveTokenText(this.#state.tokenizer, tokens)
|
|
483
|
+
);
|
|
484
|
+
log.debug('Pipeline', `After rep penalty top-5: ${topAfterPenalty.map(t => `"${t.text}"(${(t.prob * 100).toFixed(1)}%)`).join(', ')}`);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
let firstToken;
|
|
488
|
+
try {
|
|
489
|
+
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
490
|
+
} catch (error) {
|
|
491
|
+
if (!shouldRetryWithFinitenessFallback(error)) {
|
|
492
|
+
throw error;
|
|
493
|
+
}
|
|
494
|
+
log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
|
|
495
|
+
prefillLogits = await this._retryWithFinitenessFallback(
|
|
496
|
+
opts,
|
|
497
|
+
'prefill-sample',
|
|
498
|
+
() => this._prefill(inputIds, opts)
|
|
499
|
+
);
|
|
500
|
+
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
if (opts.debug) {
|
|
504
|
+
const firstTokenText = resolveTokenText(this.#state.tokenizer, [firstToken], `[${firstToken}]`, (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false));
|
|
505
|
+
log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
generatedIds.push(firstToken);
|
|
509
|
+
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
510
|
+
|
|
511
|
+
const decodeToken = (tokenId) => resolveTokenText(
|
|
512
|
+
this.#state.tokenizer,
|
|
513
|
+
[tokenId],
|
|
514
|
+
`[${tokenId}]`,
|
|
515
|
+
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
516
|
+
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
517
|
+
);
|
|
518
|
+
|
|
519
|
+
yield* emitToken(this, firstToken, decodeToken);
|
|
520
|
+
|
|
521
|
+
yield* this._runDecodeLoop(generatedIds, opts, options, {
|
|
522
|
+
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
523
|
+
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
524
|
+
stopSequenceStart: inputIds.length,
|
|
525
|
+
decodeToken,
|
|
526
|
+
logBatchPath: opts.debug,
|
|
527
|
+
emitMode: mode,
|
|
528
|
+
});
|
|
529
|
+
const tokensGenerated = this.#state.stats.decodeTokens;
|
|
530
|
+
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
531
|
+
|
|
532
|
+
if (opts.debug) {
|
|
533
|
+
log.debug('Pipeline', `Generated ${tokensGenerated} tokens in ${this.#state.stats.totalTimeMs.toFixed(0)}ms`);
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
const ttft = this.#state.stats.ttftMs ?? this.#state.stats.prefillTimeMs;
|
|
537
|
+
const decodeTokens = Math.max(0, tokensGenerated - 1);
|
|
538
|
+
const decodeSpeed = decodeTokens > 0 ? (decodeTokens / this.#state.stats.decodeTimeMs * 1000) : 0;
|
|
539
|
+
const loadMs = this.#state.stats.modelLoadMs;
|
|
540
|
+
const loadLabel = Number.isFinite(loadMs) ? `Load: ${loadMs.toFixed(0)}ms | ` : '';
|
|
541
|
+
if (opts.benchmark) {
|
|
542
|
+
log.info('Benchmark', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
543
|
+
} else {
|
|
544
|
+
log.info('Perf', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
|
|
545
|
+
}
|
|
546
|
+
trace.perf('Decode summary', {
|
|
547
|
+
ttftMs: ttft,
|
|
548
|
+
prefillMs: this.#state.stats.prefillTimeMs,
|
|
549
|
+
decodeMs: this.#state.stats.decodeTimeMs,
|
|
550
|
+
decodeTokens,
|
|
551
|
+
decodeSpeed,
|
|
552
|
+
totalMs: this.#state.stats.totalTimeMs,
|
|
553
|
+
});
|
|
554
|
+
} finally {
|
|
555
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
556
|
+
resetActiveExecutionPlan(this.#state);
|
|
557
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
558
|
+
? {
|
|
559
|
+
enabled: true,
|
|
560
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
561
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
562
|
+
}
|
|
563
|
+
: null;
|
|
564
|
+
this.#state.operatorDiagnostics = null;
|
|
565
|
+
this.#state.isGenerating = false;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
_beginFinitenessFallback(opts, reasonLabel) {
|
|
570
|
+
const originalPlan = resolveActiveExecutionPlan(this.#state);
|
|
571
|
+
const original = {
|
|
572
|
+
activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
|
|
573
|
+
seed: opts.seed,
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
const fallbackPlan = activateFallbackExecutionPlan(this.#state);
|
|
577
|
+
if (!fallbackPlan) {
|
|
578
|
+
throw new Error('[Pipeline] Finiteness fallback plan is unavailable for this model/runtime configuration.');
|
|
579
|
+
}
|
|
580
|
+
log.warn(
|
|
581
|
+
'Pipeline',
|
|
582
|
+
`FinitenessGuard fallback (${reasonLabel}): ` +
|
|
583
|
+
`${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
|
|
584
|
+
);
|
|
585
|
+
|
|
586
|
+
this.#state.decodeBuffers?.ensureBuffers({
|
|
587
|
+
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
588
|
+
intermediateSize: this.#state.modelConfig.intermediateSize,
|
|
589
|
+
activationDtype: fallbackPlan.activationDtype,
|
|
590
|
+
enablePingPong: true,
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
if (opts.seed == null) {
|
|
594
|
+
const fallbackSeedBase = (this.#state.decodeStepCount + this.#state.currentSeqLen + 1) >>> 0;
|
|
595
|
+
opts.seed = (fallbackSeedBase * 2654435761) >>> 0;
|
|
596
|
+
}
|
|
597
|
+
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
598
|
+
if (this.#state.stats.executionPlan) {
|
|
599
|
+
this.#state.stats.executionPlan.finalActivePlanId = fallbackPlan.id;
|
|
600
|
+
this.#state.stats.executionPlan.transitions.push({
|
|
601
|
+
kind: 'activate-finiteness-fallback',
|
|
602
|
+
reason: reasonLabel ?? null,
|
|
603
|
+
decodeStep: this.#state.decodeStepCount,
|
|
604
|
+
seqLen: this.#state.currentSeqLen,
|
|
605
|
+
fromPlanId: originalPlan.id,
|
|
606
|
+
toPlanId: fallbackPlan.id,
|
|
607
|
+
fromKernelPathId: originalPlan.kernelPathId ?? null,
|
|
608
|
+
toKernelPathId: fallbackPlan.kernelPathId ?? null,
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
this.#state.stats.kernelPathId = fallbackPlan.kernelPathId ?? null;
|
|
612
|
+
this.#state.stats.kernelPathSource = fallbackPlan.kernelPathSource ?? 'none';
|
|
613
|
+
|
|
614
|
+
return original;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
_endFinitenessFallback(opts, original) {
|
|
618
|
+
opts.seed = original.seed;
|
|
619
|
+
setActiveExecutionPlan(this.#state, original.activePlanId);
|
|
620
|
+
opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
|
|
621
|
+
const restoredPlan = resolveActiveExecutionPlan(this.#state);
|
|
622
|
+
if (this.#state.stats.executionPlan) {
|
|
623
|
+
this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
|
|
624
|
+
this.#state.stats.executionPlan.transitions.push({
|
|
625
|
+
kind: 'restore-primary-plan',
|
|
626
|
+
reason: null,
|
|
627
|
+
decodeStep: this.#state.decodeStepCount,
|
|
628
|
+
seqLen: this.#state.currentSeqLen,
|
|
629
|
+
fromPlanId: this.#state.executionPlanState?.fallbackPlan?.id ?? null,
|
|
630
|
+
toPlanId: restoredPlan.id,
|
|
631
|
+
fromKernelPathId: this.#state.executionPlanState?.fallbackPlan?.kernelPathId ?? null,
|
|
632
|
+
toKernelPathId: restoredPlan.kernelPathId ?? null,
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
this.#state.stats.kernelPathId = restoredPlan.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
|
|
636
|
+
this.#state.stats.kernelPathSource = restoredPlan.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
|
|
637
|
+
const nextActivationDtype = this._getEffectiveActivationDtype();
|
|
638
|
+
this.#state.decodeBuffers?.ensureBuffers({
|
|
639
|
+
hiddenSize: this.#state.modelConfig.hiddenSize,
|
|
640
|
+
intermediateSize: this.#state.modelConfig.intermediateSize,
|
|
641
|
+
activationDtype: nextActivationDtype,
|
|
642
|
+
enablePingPong: true,
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
async _retryWithFinitenessFallback(opts, reasonLabel, retryFn) {
|
|
647
|
+
if (this._hasFinitenessFallbackWindow()) {
|
|
648
|
+
return retryFn();
|
|
649
|
+
}
|
|
650
|
+
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
651
|
+
const original = this._beginFinitenessFallback(opts, reasonLabel);
|
|
652
|
+
try {
|
|
653
|
+
return await retryFn();
|
|
654
|
+
} finally {
|
|
655
|
+
this._endFinitenessFallback(opts, original);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
|
|
660
|
+
const windowTokens = this._resolveDeferredRoundingWindowTokens();
|
|
661
|
+
if (windowTokens <= 1) {
|
|
662
|
+
return this._retryWithFinitenessFallback(
|
|
663
|
+
opts,
|
|
664
|
+
reasonLabel,
|
|
665
|
+
() => this._decodeStep(generatedIds, opts)
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
this.#state.kvCache?.truncate(this.#state.currentSeqLen);
|
|
670
|
+
this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
|
|
671
|
+
try {
|
|
672
|
+
return await this._decodeStep(generatedIds, opts);
|
|
673
|
+
} catch (error) {
|
|
674
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
675
|
+
throw error;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// ==========================================================================
|
|
680
|
+
// Generation Public API
|
|
681
|
+
// ==========================================================================
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
async *generate(prompt, options = {}) {
|
|
685
|
+
yield* this._generateTokensInternal(prompt, options, 'text');
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
async *generateTokens(prompt, options = {}) {
|
|
689
|
+
yield* this._generateTokensInternal(prompt, options, 'token');
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
async generateTokenIds(prompt, options = {}) {
|
|
693
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
694
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
695
|
+
|
|
696
|
+
validateCallTimeOptions(options);
|
|
697
|
+
|
|
698
|
+
this.#state.isGenerating = true;
|
|
699
|
+
this._resetDecodeRuntimeState();
|
|
700
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
701
|
+
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
702
|
+
this.#state.stats.decodeRecordMs = 0;
|
|
703
|
+
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
704
|
+
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
705
|
+
this.#state.stats.singleTokenSubmitWaitMs = 0;
|
|
706
|
+
this.#state.stats.singleTokenReadbackWaitMs = 0;
|
|
707
|
+
this.#state.stats.singleTokenOrchestrationMs = 0;
|
|
708
|
+
this.#state.stats.decodeMode = null;
|
|
709
|
+
this.#state.stats.batchGuardReason = null;
|
|
710
|
+
this.#state.stats.ttftMs = 0;
|
|
711
|
+
const startTime = performance.now();
|
|
712
|
+
const opts = resolveGenerateOptions(this.#state, options);
|
|
713
|
+
// Validate and normalize sampling parameters through single source of truth
|
|
714
|
+
const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
|
|
715
|
+
opts.temperature = samplingConfig.temperature;
|
|
716
|
+
opts.topP = samplingConfig.topP;
|
|
717
|
+
opts.topK = samplingConfig.topK;
|
|
718
|
+
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
719
|
+
|
|
720
|
+
try {
|
|
721
|
+
const prefillStart = performance.now();
|
|
722
|
+
const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
|
|
723
|
+
let prefillLogits = initialPrefillLogits;
|
|
724
|
+
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
725
|
+
this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
|
|
726
|
+
const generatedIds = [...inputIds];
|
|
727
|
+
this.#state.stats.prefillTokens = inputIds.length;
|
|
728
|
+
|
|
729
|
+
let firstToken;
|
|
730
|
+
try {
|
|
731
|
+
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
732
|
+
} catch (error) {
|
|
733
|
+
if (!shouldRetryWithFinitenessFallback(error)) {
|
|
734
|
+
throw error;
|
|
735
|
+
}
|
|
736
|
+
prefillLogits = await this._retryWithFinitenessFallback(
|
|
737
|
+
opts,
|
|
738
|
+
'prefill-sample',
|
|
739
|
+
() => this._prefill(inputIds, opts)
|
|
740
|
+
);
|
|
741
|
+
firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
generatedIds.push(firstToken);
|
|
745
|
+
const tokenIds = [firstToken];
|
|
746
|
+
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
747
|
+
|
|
748
|
+
const stopTokenIds = this.#state.modelConfig.stopTokenIds;
|
|
749
|
+
const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
|
|
750
|
+
const stopSequenceStart = inputIds.length;
|
|
751
|
+
markKernelCacheWarmed();
|
|
752
|
+
const decodeStart = performance.now();
|
|
753
|
+
|
|
754
|
+
while (tokenIds.length < opts.maxTokens) {
|
|
755
|
+
if (options.signal?.aborted) break;
|
|
756
|
+
let nextToken;
|
|
757
|
+
try {
|
|
758
|
+
nextToken = await this._decodeNextTokenViaLogits(generatedIds, opts);
|
|
759
|
+
} catch (error) {
|
|
760
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
761
|
+
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
762
|
+
generatedIds,
|
|
763
|
+
opts,
|
|
764
|
+
`decode-step-${tokenIds.length}`
|
|
765
|
+
);
|
|
766
|
+
} else {
|
|
767
|
+
throw error;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
generatedIds.push(nextToken);
|
|
771
|
+
tokenIds.push(nextToken);
|
|
772
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
773
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
774
|
+
break;
|
|
775
|
+
}
|
|
776
|
+
if (opts.stopSequences.length > 0) {
|
|
777
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
778
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
783
|
+
this.#state.stats.tokensGenerated = tokenIds.length;
|
|
784
|
+
this.#state.stats.decodeTokens = tokenIds.length;
|
|
785
|
+
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
786
|
+
|
|
787
|
+
return {
|
|
788
|
+
tokenIds,
|
|
789
|
+
stats: this.#state.stats,
|
|
790
|
+
};
|
|
791
|
+
} finally {
|
|
792
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
793
|
+
if (this.#state.stats.executionPlan) {
|
|
794
|
+
this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
|
|
795
|
+
}
|
|
796
|
+
resetActiveExecutionPlan(this.#state);
|
|
797
|
+
this.#state.isGenerating = false;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
async prefillKVOnly(prompt, options = {}) {
|
|
803
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
804
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
805
|
+
throw new Error('Generation already in progress');
|
|
806
|
+
}
|
|
807
|
+
this._resetDecodeRuntimeState();
|
|
808
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
809
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
810
|
+
const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, 'prefillKVOnly');
|
|
811
|
+
if (opts.debug) {
|
|
812
|
+
log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
let prefillResult;
|
|
816
|
+
try {
|
|
817
|
+
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
818
|
+
} catch (error) {
|
|
819
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
820
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
|
|
821
|
+
prefillResult = await this._retryWithFinitenessFallback(
|
|
822
|
+
opts,
|
|
823
|
+
'prefillKVOnly',
|
|
824
|
+
() => this._prefillToHidden(inputIds, opts)
|
|
825
|
+
);
|
|
826
|
+
} else {
|
|
827
|
+
throw error;
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
const {
|
|
832
|
+
numTokens,
|
|
833
|
+
startPos,
|
|
834
|
+
currentRecorder,
|
|
835
|
+
recordProfile,
|
|
836
|
+
currentHiddenBuffer,
|
|
837
|
+
} = prefillResult;
|
|
838
|
+
|
|
839
|
+
// Ensure prefill work completes before returning a usable snapshot.
|
|
840
|
+
if (currentRecorder) {
|
|
841
|
+
await currentRecorder.submitAndWait();
|
|
842
|
+
await recordProfile(currentRecorder);
|
|
843
|
+
} else {
|
|
844
|
+
const device = getDevice();
|
|
845
|
+
if (device) {
|
|
846
|
+
await device.queue.onSubmittedWorkDone();
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
851
|
+
releaseBuffer(currentHiddenBuffer);
|
|
852
|
+
|
|
853
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
854
|
+
if (!snapshot) {
|
|
855
|
+
throw new Error('KV cache unavailable after prefill');
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
return {
|
|
859
|
+
cache: snapshot,
|
|
860
|
+
seqLen: this.#state.currentSeqLen,
|
|
861
|
+
tokens: inputIds,
|
|
862
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
async prefillWithEmbedding(prompt, options = {}) {
|
|
867
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
868
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
869
|
+
throw new Error('Generation already in progress');
|
|
870
|
+
}
|
|
871
|
+
this._resetDecodeRuntimeState();
|
|
872
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
873
|
+
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
874
|
+
const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding');
|
|
875
|
+
if (opts.debug) {
|
|
876
|
+
log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
let prefillResult;
|
|
880
|
+
try {
|
|
881
|
+
prefillResult = await this._prefillToHidden(inputIds, opts);
|
|
882
|
+
} catch (error) {
|
|
883
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
884
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
|
|
885
|
+
prefillResult = await this._retryWithFinitenessFallback(
|
|
886
|
+
opts,
|
|
887
|
+
'prefillWithEmbedding',
|
|
888
|
+
() => this._prefillToHidden(inputIds, opts)
|
|
889
|
+
);
|
|
890
|
+
} else {
|
|
891
|
+
throw error;
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const {
|
|
896
|
+
numTokens,
|
|
897
|
+
config,
|
|
898
|
+
startPos,
|
|
899
|
+
activationDtype,
|
|
900
|
+
activationBytes,
|
|
901
|
+
currentRecorder,
|
|
902
|
+
recordProfile,
|
|
903
|
+
currentHiddenBuffer,
|
|
904
|
+
} = prefillResult;
|
|
905
|
+
|
|
906
|
+
// Ensure prefill work completes before readback.
|
|
907
|
+
if (currentRecorder) {
|
|
908
|
+
await currentRecorder.submitAndWait();
|
|
909
|
+
await recordProfile(currentRecorder);
|
|
910
|
+
} else {
|
|
911
|
+
const device = getDevice();
|
|
912
|
+
if (device) {
|
|
913
|
+
await device.queue.onSubmittedWorkDone();
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
if (!allowReadback('pipeline.prefill.embedding')) {
|
|
918
|
+
throw new Error('GPU readback disabled; cannot return embedding');
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
let embedding;
|
|
922
|
+
try {
|
|
923
|
+
const hiddenSize = config.hiddenSize;
|
|
924
|
+
const hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
925
|
+
const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
|
|
926
|
+
if (hiddenData.byteLength === 0) {
|
|
927
|
+
throw new Error('GPU readback disabled; cannot return embedding');
|
|
928
|
+
}
|
|
929
|
+
const hiddenStates = decodeReadback(hiddenData, activationDtype);
|
|
930
|
+
const finalNormWeights = await this._getFinalNormWeights();
|
|
931
|
+
embedding = this._extractEmbeddingFromHidden(
|
|
932
|
+
hiddenStates,
|
|
933
|
+
numTokens,
|
|
934
|
+
hiddenSize,
|
|
935
|
+
opts.embeddingMode,
|
|
936
|
+
finalNormWeights,
|
|
937
|
+
config
|
|
938
|
+
);
|
|
939
|
+
} finally {
|
|
940
|
+
releaseBuffer(currentHiddenBuffer);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
944
|
+
|
|
945
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
946
|
+
if (!snapshot) {
|
|
947
|
+
throw new Error('KV cache unavailable after prefill');
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
return {
|
|
951
|
+
cache: snapshot,
|
|
952
|
+
seqLen: this.#state.currentSeqLen,
|
|
953
|
+
tokens: inputIds,
|
|
954
|
+
embedding,
|
|
955
|
+
embeddingMode: opts.embeddingMode,
|
|
956
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
957
|
+
};
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
async prefillWithLogits(prompt, options = {}) {
|
|
961
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
962
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
963
|
+
throw new Error('Generation already in progress');
|
|
964
|
+
}
|
|
965
|
+
this._resetDecodeRuntimeState();
|
|
966
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
967
|
+
const opts = resolvePrefillOptions(this.#state, options);
|
|
968
|
+
const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
969
|
+
|
|
970
|
+
const snapshot = this.#state.kvCache?.clone();
|
|
971
|
+
if (!snapshot) {
|
|
972
|
+
throw new Error('KV cache unavailable after prefill');
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
return {
|
|
976
|
+
cache: snapshot,
|
|
977
|
+
seqLen: this.#state.currentSeqLen,
|
|
978
|
+
tokens: inputIds,
|
|
979
|
+
logits,
|
|
980
|
+
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
async *generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
986
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
987
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
988
|
+
|
|
989
|
+
validateCallTimeOptions(options);
|
|
990
|
+
|
|
991
|
+
// Apply snapshot
|
|
992
|
+
this.#state.kvCache = prefix.cache.clone();
|
|
993
|
+
if (this.#state.useGPU && this.#state.kvCache) {
|
|
994
|
+
const device = getDevice();
|
|
995
|
+
if (device) {
|
|
996
|
+
this.#state.kvCache.setGPUContext({ device });
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
if (
|
|
1000
|
+
hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
|
|
1001
|
+
&& prefix.linearAttention == null
|
|
1002
|
+
) {
|
|
1003
|
+
throw new Error(
|
|
1004
|
+
'Prefix snapshot is missing linear_attention recurrent state. ' +
|
|
1005
|
+
'Regenerate the prefix snapshot using the current runtime.'
|
|
1006
|
+
);
|
|
1007
|
+
}
|
|
1008
|
+
this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
|
|
1009
|
+
this.#state.linearAttentionRuntime,
|
|
1010
|
+
prefix.linearAttention ?? null
|
|
1011
|
+
);
|
|
1012
|
+
this.#state.currentSeqLen = prefix.seqLen;
|
|
1013
|
+
|
|
1014
|
+
this.#state.isGenerating = true;
|
|
1015
|
+
this.#state.decodeStepCount = 0;
|
|
1016
|
+
resetActiveExecutionPlan(this.#state);
|
|
1017
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1018
|
+
this.#state.stats.gpuTimeDecodeMs = undefined;
|
|
1019
|
+
this.#state.decodeRing?.reset();
|
|
1020
|
+
this.#state.stats.decodeRecordMs = 0;
|
|
1021
|
+
this.#state.stats.decodeSubmitWaitMs = 0;
|
|
1022
|
+
this.#state.stats.decodeReadbackWaitMs = 0;
|
|
1023
|
+
this.#state.stats.ttftMs = 0;
|
|
1024
|
+
const startTime = performance.now();
|
|
1025
|
+
|
|
1026
|
+
const opts = resolveGenerateOptions(this.#state, options);
|
|
1027
|
+
|
|
1028
|
+
try {
|
|
1029
|
+
const processedPrompt = resolvePromptInput(this.#state, prompt, opts.useChatTemplate, 'generateWithPrefixKV');
|
|
1030
|
+
|
|
1031
|
+
const inputIds = this.#state.tokenizer.encode(processedPrompt);
|
|
1032
|
+
this._assertTokenIdsInRange(inputIds, 'generateWithPrefixKV.encode');
|
|
1033
|
+
const generatedIds = [...prefix.tokens, ...inputIds];
|
|
1034
|
+
const promptTokenCount = generatedIds.length;
|
|
1035
|
+
this.#state.stats.prefillTokens = inputIds.length;
|
|
1036
|
+
|
|
1037
|
+
const prefillStart = performance.now();
|
|
1038
|
+
const prefillLogits = await this._prefill(inputIds, opts);
|
|
1039
|
+
this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
1040
|
+
|
|
1041
|
+
applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
|
|
1042
|
+
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
1043
|
+
const firstToken = sample(prefillLogits, {
|
|
1044
|
+
temperature: opts.temperature,
|
|
1045
|
+
topP: opts.topP,
|
|
1046
|
+
topK: opts.topK,
|
|
1047
|
+
padTokenId,
|
|
1048
|
+
seed: opts.seed,
|
|
1049
|
+
});
|
|
1050
|
+
|
|
1051
|
+
generatedIds.push(firstToken);
|
|
1052
|
+
this.#state.stats.ttftMs = performance.now() - startTime;
|
|
1053
|
+
|
|
1054
|
+
const firstText = resolveTokenText(
|
|
1055
|
+
this.#state.tokenizer,
|
|
1056
|
+
[firstToken],
|
|
1057
|
+
`[${firstToken}]`,
|
|
1058
|
+
(tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
|
|
1059
|
+
(tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
|
|
1060
|
+
);
|
|
1061
|
+
yield firstText;
|
|
1062
|
+
if (options.onToken) options.onToken(firstToken, firstText);
|
|
1063
|
+
|
|
1064
|
+
yield* this._runDecodeLoop(generatedIds, opts, options, {
|
|
1065
|
+
stopTokenIds: this.#state.modelConfig.stopTokenIds,
|
|
1066
|
+
eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
|
|
1067
|
+
stopSequenceStart: promptTokenCount,
|
|
1068
|
+
decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
|
|
1069
|
+
logBatchPath: false,
|
|
1070
|
+
});
|
|
1071
|
+
this.#state.stats.totalTimeMs = performance.now() - startTime;
|
|
1072
|
+
} finally {
|
|
1073
|
+
this._closeFinitenessFallbackWindow(opts);
|
|
1074
|
+
resetActiveExecutionPlan(this.#state);
|
|
1075
|
+
this.#state.isGenerating = false;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
// ==========================================================================
|
|
1080
|
+
// Internal Methods (Prefill, Decode, Helpers)
|
|
1081
|
+
// ==========================================================================
|
|
1082
|
+
|
|
1083
|
+
async *_runDecodeLoop(generatedIds, opts, options, runtime) {
|
|
1084
|
+
const {
|
|
1085
|
+
stopTokenIds,
|
|
1086
|
+
eosToken,
|
|
1087
|
+
stopSequenceStart,
|
|
1088
|
+
decodeToken,
|
|
1089
|
+
logBatchPath = false,
|
|
1090
|
+
emitMode = 'text',
|
|
1091
|
+
} = runtime;
|
|
1092
|
+
|
|
1093
|
+
let tokensGenerated = 1;
|
|
1094
|
+
markKernelCacheWarmed();
|
|
1095
|
+
|
|
1096
|
+
const decodeStart = performance.now();
|
|
1097
|
+
const lmHead = this.#state.weights.get('lm_head');
|
|
1098
|
+
const embedBuffer = this.#state.weights.get('embed');
|
|
1099
|
+
const hasCpuWeights = isCpuWeightBuffer(lmHead)
|
|
1100
|
+
|| isCpuWeightBuffer(embedBuffer)
|
|
1101
|
+
|| lmHead instanceof Float32Array
|
|
1102
|
+
|| embedBuffer instanceof Float32Array;
|
|
1103
|
+
const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
|
|
1104
|
+
const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
|
|
1105
|
+
const executionPlan = opts.executionPlan;
|
|
1106
|
+
let useBatchPath = shouldUseBatchDecode({
|
|
1107
|
+
batchSize: executionPlan.batchSize,
|
|
1108
|
+
useGPU: this.#state.useGPU,
|
|
1109
|
+
gpuSamplingAvailable,
|
|
1110
|
+
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
|
|
1111
|
+
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
1112
|
+
isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
|
|
1113
|
+
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
1114
|
+
});
|
|
1115
|
+
if (!useBatchPath) {
|
|
1116
|
+
let reason = null;
|
|
1117
|
+
if (hasCpuWeights) reason = 'cpu_weights';
|
|
1118
|
+
else if (!this.#state.useGPU) reason = 'no_gpu';
|
|
1119
|
+
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
1120
|
+
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
1121
|
+
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
1122
|
+
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
1123
|
+
else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
1124
|
+
else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
|
|
1125
|
+
this.#state.stats.decodeMode = opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token';
|
|
1126
|
+
this.#state.stats.batchGuardReason = reason;
|
|
1127
|
+
} else {
|
|
1128
|
+
this.#state.stats.decodeMode = 'batched_gpu';
|
|
1129
|
+
this.#state.stats.batchGuardReason = null;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
const readbackInterval = executionPlan.readbackInterval;
|
|
1133
|
+
const intervalBatches = readbackInterval == null ? 1 : readbackInterval;
|
|
1134
|
+
const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
1135
|
+
|
|
1136
|
+
const decodeSingleTokenViaLogits = async () => this._decodeNextTokenViaLogits(generatedIds, opts);
|
|
1137
|
+
|
|
1138
|
+
if (logBatchPath && useBatchPath) {
|
|
1139
|
+
log.debug(
|
|
1140
|
+
'Pipeline',
|
|
1141
|
+
`Using batch decode path with batchSize=${executionPlan.batchSize}, stopCheckMode=${executionPlan.stopCheckMode}, readbackInterval=${readbackInterval}`
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
while (tokensGenerated < opts.maxTokens) {
|
|
1146
|
+
if (options.signal?.aborted) break;
|
|
1147
|
+
if (this._hasFinitenessFallbackWindow() && useBatchPath) {
|
|
1148
|
+
useBatchPath = false;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
if (useBatchPath) {
|
|
1152
|
+
const remaining = opts.maxTokens - tokensGenerated;
|
|
1153
|
+
const thisBatchSize = Math.min(executionPlan.batchSize * intervalBatches, remaining);
|
|
1154
|
+
const lastToken = generatedIds[generatedIds.length - 1];
|
|
1155
|
+
|
|
1156
|
+
try {
|
|
1157
|
+
const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, opts);
|
|
1158
|
+
let batchTokens = [];
|
|
1159
|
+
let hitStop = false;
|
|
1160
|
+
for (const tokenId of batchResult.tokens) {
|
|
1161
|
+
if (isStopToken(tokenId, stopTokenIds, eosToken)) {
|
|
1162
|
+
hitStop = true;
|
|
1163
|
+
break;
|
|
1164
|
+
}
|
|
1165
|
+
generatedIds.push(tokenId);
|
|
1166
|
+
tokensGenerated++;
|
|
1167
|
+
if (emitMode === 'token') {
|
|
1168
|
+
yield tokenId;
|
|
1169
|
+
if (options.onToken) options.onToken(tokenId, '');
|
|
1170
|
+
batchTokens.push({ id: tokenId, text: '' });
|
|
1171
|
+
} else {
|
|
1172
|
+
const tokenText = decodeToken(tokenId);
|
|
1173
|
+
yield tokenText;
|
|
1174
|
+
if (options.onToken) options.onToken(tokenId, tokenText);
|
|
1175
|
+
batchTokens.push({ id: tokenId, text: tokenText });
|
|
1176
|
+
}
|
|
1177
|
+
if (batchTokens.length === executionPlan.batchSize) {
|
|
1178
|
+
if (options.onBatch) options.onBatch(batchTokens);
|
|
1179
|
+
batchTokens = [];
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
|
|
1183
|
+
if (hitStop || batchResult.actualCount < thisBatchSize) break;
|
|
1184
|
+
if (opts.stopSequences.length > 0) {
|
|
1185
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1186
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1187
|
+
}
|
|
1188
|
+
} catch (error) {
|
|
1189
|
+
log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
|
|
1190
|
+
useBatchPath = false;
|
|
1191
|
+
let nextToken;
|
|
1192
|
+
try {
|
|
1193
|
+
nextToken = await decodeSingleTokenViaLogits();
|
|
1194
|
+
} catch (singleTokenError) {
|
|
1195
|
+
if (shouldRetryWithFinitenessFallback(singleTokenError)) {
|
|
1196
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
1197
|
+
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
1198
|
+
generatedIds,
|
|
1199
|
+
opts,
|
|
1200
|
+
`decode-batch-step-${tokensGenerated}`
|
|
1201
|
+
);
|
|
1202
|
+
} else {
|
|
1203
|
+
throw singleTokenError;
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
generatedIds.push(nextToken);
|
|
1207
|
+
tokensGenerated++;
|
|
1208
|
+
if (emitMode === 'token') {
|
|
1209
|
+
yield nextToken;
|
|
1210
|
+
if (options.onToken) options.onToken(nextToken, '');
|
|
1211
|
+
} else {
|
|
1212
|
+
const tokenText = decodeToken(nextToken);
|
|
1213
|
+
yield tokenText;
|
|
1214
|
+
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
1215
|
+
}
|
|
1216
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
1217
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
|
|
1218
|
+
}
|
|
1219
|
+
} else if (opts.speculation?.mode === 'self') {
|
|
1220
|
+
// Self-speculation: decode 2 tokens per iteration (base + speculative).
|
|
1221
|
+
// Same-model speculation always accepts under greedy because the model
|
|
1222
|
+
// is deterministic — both base and speculative use the same weights and
|
|
1223
|
+
// state. The benefit is amortizing per-iteration overhead for models
|
|
1224
|
+
// where batch decode is disabled (e.g., linear attention).
|
|
1225
|
+
const doSpecDecode = hasLinearLayers
|
|
1226
|
+
? () => this._decodeStep(generatedIds, opts)
|
|
1227
|
+
: decodeSingleTokenViaLogits;
|
|
1228
|
+
|
|
1229
|
+
// Base decode
|
|
1230
|
+
let baseToken;
|
|
1231
|
+
try {
|
|
1232
|
+
baseToken = await doSpecDecode();
|
|
1233
|
+
} catch (error) {
|
|
1234
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
1235
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
|
|
1236
|
+
baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
|
|
1237
|
+
} else {
|
|
1238
|
+
throw error;
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
generatedIds.push(baseToken);
|
|
1242
|
+
tokensGenerated++;
|
|
1243
|
+
if (emitMode === 'token') {
|
|
1244
|
+
yield baseToken;
|
|
1245
|
+
if (options.onToken) options.onToken(baseToken, '');
|
|
1246
|
+
} else {
|
|
1247
|
+
const text = decodeToken(baseToken);
|
|
1248
|
+
yield text;
|
|
1249
|
+
if (options.onToken) options.onToken(baseToken, text);
|
|
1250
|
+
}
|
|
1251
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
1252
|
+
|
|
1253
|
+
if (isStopToken(baseToken, stopTokenIds, eosToken)) break;
|
|
1254
|
+
if (tokensGenerated >= opts.maxTokens) break;
|
|
1255
|
+
if (opts.stopSequences.length > 0) {
|
|
1256
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1257
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
// Speculative decode (tokens=1)
|
|
1261
|
+
let specToken;
|
|
1262
|
+
try {
|
|
1263
|
+
specToken = await doSpecDecode();
|
|
1264
|
+
} catch (error) {
|
|
1265
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
1266
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
|
|
1267
|
+
specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
|
|
1268
|
+
} else {
|
|
1269
|
+
throw error;
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
generatedIds.push(specToken);
|
|
1273
|
+
tokensGenerated++;
|
|
1274
|
+
this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
|
|
1275
|
+
this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
|
|
1276
|
+
if (emitMode === 'token') {
|
|
1277
|
+
yield specToken;
|
|
1278
|
+
if (options.onToken) options.onToken(specToken, '');
|
|
1279
|
+
} else {
|
|
1280
|
+
const text = decodeToken(specToken);
|
|
1281
|
+
yield text;
|
|
1282
|
+
if (options.onToken) options.onToken(specToken, text);
|
|
1283
|
+
}
|
|
1284
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
1285
|
+
|
|
1286
|
+
if (opts.debug || opts.benchmark) {
|
|
1287
|
+
const elapsedMs = performance.now() - decodeStart;
|
|
1288
|
+
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
1289
|
+
log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
if (isStopToken(specToken, stopTokenIds, eosToken)) break;
|
|
1293
|
+
if (opts.stopSequences.length > 0) {
|
|
1294
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1295
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1296
|
+
}
|
|
1297
|
+
} else {
|
|
1298
|
+
const tokenStart = performance.now();
|
|
1299
|
+
let nextToken;
|
|
1300
|
+
try {
|
|
1301
|
+
nextToken = hasLinearLayers
|
|
1302
|
+
? await this._decodeStep(generatedIds, opts)
|
|
1303
|
+
: await decodeSingleTokenViaLogits();
|
|
1304
|
+
} catch (error) {
|
|
1305
|
+
if (shouldRetryWithFinitenessFallback(error)) {
|
|
1306
|
+
log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
|
|
1307
|
+
nextToken = await this._retryDecodeStepWithFinitenessWindow(
|
|
1308
|
+
generatedIds,
|
|
1309
|
+
opts,
|
|
1310
|
+
`decode-step-${tokensGenerated}`
|
|
1311
|
+
);
|
|
1312
|
+
} else {
|
|
1313
|
+
throw error;
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
const tokenTime = performance.now() - tokenStart;
|
|
1317
|
+
generatedIds.push(nextToken);
|
|
1318
|
+
tokensGenerated++;
|
|
1319
|
+
const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
|
|
1320
|
+
if (emitMode === 'token') {
|
|
1321
|
+
yield nextToken;
|
|
1322
|
+
if (options.onToken) options.onToken(nextToken, '');
|
|
1323
|
+
} else {
|
|
1324
|
+
yield tokenText;
|
|
1325
|
+
if (options.onToken) options.onToken(nextToken, tokenText);
|
|
1326
|
+
}
|
|
1327
|
+
this._consumeFinitenessFallbackToken(opts);
|
|
1328
|
+
|
|
1329
|
+
if (opts.debug || opts.benchmark) {
|
|
1330
|
+
const elapsedMs = performance.now() - decodeStart;
|
|
1331
|
+
const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
|
|
1332
|
+
log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
|
|
1336
|
+
if (opts.stopSequences.length > 0) {
|
|
1337
|
+
const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
|
|
1338
|
+
if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
|
|
1344
|
+
this.#state.stats.tokensGenerated = tokensGenerated;
|
|
1345
|
+
this.#state.stats.decodeTokens = tokensGenerated;
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
async _prefillToHidden(inputIds, opts) {
|
|
1349
|
+
// Internal-only: reuse the main prefill implementation but stop before logits.
|
|
1350
|
+
return this._prefill(inputIds, { ...opts, _returnHidden: true });
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
async _prefill(inputIds, opts) {
|
|
1355
|
+
const numTokens = inputIds.length;
|
|
1356
|
+
const config = this.#state.modelConfig;
|
|
1357
|
+
const startPos = this.#state.currentSeqLen;
|
|
1358
|
+
const returnHidden = opts?._returnHidden === true;
|
|
1359
|
+
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
1360
|
+
|
|
1361
|
+
if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
|
|
1362
|
+
this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
|
|
1363
|
+
}
|
|
1364
|
+
if (startPos === 0) {
|
|
1365
|
+
for (const [, convState] of this.#state.convLayerStates) {
|
|
1366
|
+
if (convState.convStateGPU && convState.hiddenSize && convState.kernelSize) {
|
|
1367
|
+
uploadData(convState.convStateGPU, new Float32Array(convState.hiddenSize * (convState.kernelSize - 1)));
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
const embedBufferRaw = this.#state.weights.get('embed');
|
|
1373
|
+
if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
|
|
1374
|
+
throw new Error('Embed buffer not found or not a supported buffer type');
|
|
1375
|
+
}
|
|
1376
|
+
const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
|
|
1377
|
+
const embedDtype = isCpuWeightBuffer(embedBufferRaw)
|
|
1378
|
+
? embedBufferRaw.dtype
|
|
1379
|
+
: getWeightDtype(embedBufferRaw);
|
|
1380
|
+
if (opts.debug) {
|
|
1381
|
+
const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
|
|
1382
|
+
log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
const device = getDevice();
|
|
1386
|
+
const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
|
|
1387
|
+
const disableCommandBatching = opts.disableCommandBatching === true
|
|
1388
|
+
|| opts.debug === true
|
|
1389
|
+
|| this.#state.kvCache?.layout === 'bdpa_paged';
|
|
1390
|
+
const createRecorder = (label) => {
|
|
1391
|
+
if (!device || disableCommandBatching) return undefined;
|
|
1392
|
+
return opts.profile ? createProfilingRecorder(label) : createCommandRecorder(label);
|
|
1393
|
+
};
|
|
1394
|
+
const recorder = createRecorder('prefill');
|
|
1395
|
+
const debugCheckBuffer = this.#state.debug
|
|
1396
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
1397
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
1398
|
+
: undefined;
|
|
1399
|
+
const context = buildLayerContext(
|
|
1400
|
+
this.#state,
|
|
1401
|
+
recorder,
|
|
1402
|
+
false,
|
|
1403
|
+
opts.debugLayers,
|
|
1404
|
+
debugCheckBuffer,
|
|
1405
|
+
opts.executionPlan
|
|
1406
|
+
);
|
|
1407
|
+
context.currentTokenIds = inputIds;
|
|
1408
|
+
let gpuTimePrefillMs = 0;
|
|
1409
|
+
let hasGpuTimePrefill = false;
|
|
1410
|
+
const recordProfile = async (rec) => {
|
|
1411
|
+
if (!opts.profile || !rec?.isProfilingEnabled()) return;
|
|
1412
|
+
const timings = await rec.resolveProfileTimings();
|
|
1413
|
+
const total = sumProfileTimings(timings);
|
|
1414
|
+
if (total !== null) {
|
|
1415
|
+
gpuTimePrefillMs += total;
|
|
1416
|
+
hasGpuTimePrefill = true;
|
|
1417
|
+
}
|
|
1418
|
+
if (timings) {
|
|
1419
|
+
log.warn('Profile', `Prefill (${rec.label}):`);
|
|
1420
|
+
log.warn('Profile', CommandRecorder.formatProfileReport(timings));
|
|
1421
|
+
}
|
|
1422
|
+
};
|
|
1423
|
+
|
|
1424
|
+
const benchmarkSubmits = opts.debug;
|
|
1425
|
+
if (benchmarkSubmits) {
|
|
1426
|
+
setTrackSubmits(true);
|
|
1427
|
+
resetSubmitStats();
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
|
|
1431
|
+
const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
|
|
1432
|
+
let hiddenStates = await embed(inputIds, embedBuffer, {
|
|
1433
|
+
hiddenSize: config.hiddenSize,
|
|
1434
|
+
vocabSize: config.vocabSize,
|
|
1435
|
+
scaleEmbeddings: config.scaleEmbeddings,
|
|
1436
|
+
debug: opts.debug,
|
|
1437
|
+
recorder,
|
|
1438
|
+
transpose: this.#state.embeddingTranspose,
|
|
1439
|
+
debugProbes: this.#state.runtimeConfig.shared.debug.probes,
|
|
1440
|
+
activationDtype,
|
|
1441
|
+
embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
|
|
1442
|
+
});
|
|
1443
|
+
|
|
1444
|
+
if (opts.debug && isGpuBufferInstance(hiddenStates)) {
|
|
1445
|
+
if (recorder) {
|
|
1446
|
+
await recorder.submitAndWait();
|
|
1447
|
+
await recordProfile(recorder);
|
|
1448
|
+
}
|
|
1449
|
+
const debugReadbackSize = this.#state.runtimeConfig.shared.debug.pipeline.readbackSampleSize;
|
|
1450
|
+
const sample = await readBuffer(hiddenStates, Math.min(debugReadbackSize, hiddenStates.size));
|
|
1451
|
+
const f32 = decodeReadback(sample, activationDtype);
|
|
1452
|
+
const nanCount = f32.filter(x => !Number.isFinite(x)).length;
|
|
1453
|
+
let maxAbs = 0;
|
|
1454
|
+
for (let i = 0; i < f32.length; i++) {
|
|
1455
|
+
const abs = Math.abs(f32[i]);
|
|
1456
|
+
if (abs > maxAbs) maxAbs = abs;
|
|
1457
|
+
}
|
|
1458
|
+
const first8 = Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ');
|
|
1459
|
+
log.debug('Pipeline', `After embed: buffer.label=${hiddenStates.label}, buffer.size=${hiddenStates.size}, maxAbs=${maxAbs.toFixed(4)}`);
|
|
1460
|
+
log.debug('Pipeline', `After embed first8=[${first8}], nan=${nanCount}/${f32.length}`);
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
if (opts.debug) {
|
|
1464
|
+
log.debug('Pipeline', `LAYER_LOOP_START: numLayers=${config.numLayers}, useGPU=${context.useGPU}`);
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
if (this.#state.finitenessBuffer) {
|
|
1468
|
+
const device = getDevice();
|
|
1469
|
+
if (device) {
|
|
1470
|
+
device.queue.writeBuffer(this.#state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
let currentRecorder = recorder;
|
|
1475
|
+
|
|
1476
|
+
let currentHiddenBuffer = hiddenStates.buffer;
|
|
1477
|
+
for (let l = 0; l < config.numLayers; l++) {
|
|
1478
|
+
context.recorder = currentRecorder;
|
|
1479
|
+
|
|
1480
|
+
const prevBuffer = currentHiddenBuffer;
|
|
1481
|
+
const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
|
|
1482
|
+
if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
|
|
1483
|
+
currentHiddenBuffer = layerOutput;
|
|
1484
|
+
|
|
1485
|
+
const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
|
|
1486
|
+
|
|
1487
|
+
if (isCheckpoint && currentRecorder) {
|
|
1488
|
+
await currentRecorder.submitAndWait();
|
|
1489
|
+
await recordProfile(currentRecorder);
|
|
1490
|
+
currentRecorder = undefined;
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
|
|
1494
|
+
if (shouldDebug && !currentRecorder) {
|
|
1495
|
+
const device = getDevice();
|
|
1496
|
+
if (device) {
|
|
1497
|
+
if (allowReadback(`pipeline.prefill.layer-${l}`)) {
|
|
1498
|
+
try {
|
|
1499
|
+
const sampleSize = config.hiddenSize * activationBytes;
|
|
1500
|
+
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
1501
|
+
const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
|
|
1502
|
+
const data = decodeReadback(readback, activationDtype);
|
|
1503
|
+
let min = Infinity;
|
|
1504
|
+
let max = -Infinity;
|
|
1505
|
+
let maxAbs = 0;
|
|
1506
|
+
for (const v of data) {
|
|
1507
|
+
if (!Number.isFinite(v)) continue;
|
|
1508
|
+
if (v < min) min = v;
|
|
1509
|
+
if (v > max) max = v;
|
|
1510
|
+
const av = Math.abs(v);
|
|
1511
|
+
if (av > maxAbs) maxAbs = av;
|
|
1512
|
+
}
|
|
1513
|
+
const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
|
|
1514
|
+
log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
|
|
1515
|
+
} catch (e) {
|
|
1516
|
+
log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
|
|
1523
|
+
currentRecorder = createRecorder('prefill-cont');
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
if (prevBuffer !== currentHiddenBuffer) {
|
|
1527
|
+
if (currentRecorder) {
|
|
1528
|
+
currentRecorder.trackTemporaryBuffer(prevBuffer);
|
|
1529
|
+
} else {
|
|
1530
|
+
releaseBuffer(prevBuffer);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
if (this.#state.finitenessBuffer) {
|
|
1536
|
+
if (currentRecorder) {
|
|
1537
|
+
await currentRecorder.submitAndWait();
|
|
1538
|
+
await recordProfile(currentRecorder);
|
|
1539
|
+
currentRecorder = undefined;
|
|
1540
|
+
}
|
|
1541
|
+
const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
|
|
1542
|
+
const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
|
|
1543
|
+
const finitenessStatus = parseFinitenessStatusWords(u32, 0);
|
|
1544
|
+
if (finitenessStatus.triggered) {
|
|
1545
|
+
if (isGpuBufferInstance(currentHiddenBuffer)) {
|
|
1546
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1547
|
+
}
|
|
1548
|
+
throw new FinitenessError(`F16 bounds exceeded during prefill${finitenessStatus.metadata}`);
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
if (benchmarkSubmits) {
|
|
1553
|
+
logSubmitStats(`Prefill (${numTokens} tokens, ${config.numLayers} layers)`);
|
|
1554
|
+
setTrackSubmits(false);
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
if (opts.debug) {
|
|
1558
|
+
log.debug('Pipeline', `LAYER_LOOP_DONE, currentHiddenBuffer type=${currentHiddenBuffer?.constructor?.name}`);
|
|
1559
|
+
if (currentHiddenBuffer && allowReadback('pipeline.prefill.final-hidden')) {
|
|
1560
|
+
const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
|
|
1561
|
+
const sampleSize = config.hiddenSize * activationBytes;
|
|
1562
|
+
const data = decodeReadback(
|
|
1563
|
+
await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize),
|
|
1564
|
+
activationDtype
|
|
1565
|
+
);
|
|
1566
|
+
const nanCount = Array.from(data).filter(x => !Number.isFinite(x)).length;
|
|
1567
|
+
const nonZero = Array.from(data).filter(x => Number.isFinite(x) && x !== 0).slice(0, 5);
|
|
1568
|
+
log.debug('Pipeline', `FINAL_HIDDEN[pos=${numTokens - 1}]: nan=${nanCount}/${data.length}, sample=[${nonZero.map(x => x.toFixed(4)).join(', ')}]`);
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
if (hasGpuTimePrefill) {
|
|
1573
|
+
this.#state.stats.gpuTimePrefillMs = gpuTimePrefillMs;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
if (returnHidden) {
|
|
1577
|
+
return {
|
|
1578
|
+
numTokens,
|
|
1579
|
+
config,
|
|
1580
|
+
startPos,
|
|
1581
|
+
activationDtype,
|
|
1582
|
+
activationBytes,
|
|
1583
|
+
currentRecorder,
|
|
1584
|
+
recordProfile,
|
|
1585
|
+
debugCheckBuffer,
|
|
1586
|
+
currentHiddenBuffer,
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
|
|
1591
|
+
let lastLogits;
|
|
1592
|
+
let logitsVocabSize = config.vocabSize;
|
|
1593
|
+
let usedRecordedLogits = false;
|
|
1594
|
+
const lmHead = this.#state.weights.get('lm_head');
|
|
1595
|
+
const canRecordLogits = !!currentRecorder
|
|
1596
|
+
&& !!lmHead
|
|
1597
|
+
&& !isCpuWeightBuffer(lmHead)
|
|
1598
|
+
&& !this.#state.disableRecordedLogits
|
|
1599
|
+
&& numTokens === 1;
|
|
1600
|
+
if (currentRecorder && canRecordLogits) {
|
|
1601
|
+
const recorded = await recordLogitsGPU(
|
|
1602
|
+
currentRecorder,
|
|
1603
|
+
currentHiddenBuffer,
|
|
1604
|
+
numTokens,
|
|
1605
|
+
getLogitsWeights(this.#state),
|
|
1606
|
+
getLogitsConfig(this.#state)
|
|
1607
|
+
);
|
|
1608
|
+
logitsVocabSize = recorded.vocabSize;
|
|
1609
|
+
usedRecordedLogits = true;
|
|
1610
|
+
|
|
1611
|
+
await currentRecorder.submitAndWait();
|
|
1612
|
+
await recordProfile(currentRecorder);
|
|
1613
|
+
|
|
1614
|
+
const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: recorded.logitsDtype });
|
|
1615
|
+
const lastLogitsSize = logitsVocabSize * logitsBytes;
|
|
1616
|
+
const lastLogitsOffset = (numTokens - 1) * lastLogitsSize;
|
|
1617
|
+
const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
|
|
1618
|
+
releaseBuffer(recorded.logitsBuffer);
|
|
1619
|
+
lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
|
|
1620
|
+
|
|
1621
|
+
const health = getLogitsHealth(lastLogits);
|
|
1622
|
+
if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
|
|
1623
|
+
log.warn(
|
|
1624
|
+
'Logits',
|
|
1625
|
+
`Recorded logits invalid (nan=${health.nanCount} inf=${health.infCount} nonZero=${health.nonZeroCount}, maxAbs=${health.maxAbs.toFixed(3)}); recomputing without recorder.`
|
|
1626
|
+
);
|
|
1627
|
+
this.#state.disableRecordedLogits = true;
|
|
1628
|
+
this.#state.disableFusedDecode = true;
|
|
1629
|
+
const fallbackLogits = await computeLogits(
|
|
1630
|
+
currentHiddenBuffer,
|
|
1631
|
+
numTokens,
|
|
1632
|
+
getLogitsWeights(this.#state),
|
|
1633
|
+
getLogitsConfig(this.#state),
|
|
1634
|
+
this.#state.useGPU,
|
|
1635
|
+
this.#state.debugFlags,
|
|
1636
|
+
undefined,
|
|
1637
|
+
debugCheckBuffer,
|
|
1638
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
1639
|
+
{ lastPositionOnly: true }
|
|
1640
|
+
);
|
|
1641
|
+
const fallbackHealth = getLogitsHealth(fallbackLogits);
|
|
1642
|
+
if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
|
|
1643
|
+
throw new Error(
|
|
1644
|
+
`[Logits] Fallback logits invalid (nan=${fallbackHealth.nanCount} inf=${fallbackHealth.infCount} nonZero=${fallbackHealth.nonZeroCount}, maxAbs=${fallbackHealth.maxAbs.toFixed(3)}). ` +
|
|
1645
|
+
'This indicates upstream kernel output is NaN/Inf (often prefill attention/matmul).'
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1648
|
+
logitsVocabSize = config.vocabSize;
|
|
1649
|
+
usedRecordedLogits = false;
|
|
1650
|
+
lastLogits = fallbackLogits.length === logitsVocabSize
|
|
1651
|
+
? fallbackLogits
|
|
1652
|
+
: extractLastPositionLogits(fallbackLogits, numTokens, logitsVocabSize);
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1656
|
+
} else {
|
|
1657
|
+
if (currentRecorder) {
|
|
1658
|
+
await currentRecorder.submitAndWait();
|
|
1659
|
+
await recordProfile(currentRecorder);
|
|
1660
|
+
}
|
|
1661
|
+
const logits = await computeLogits(
|
|
1662
|
+
currentHiddenBuffer,
|
|
1663
|
+
numTokens,
|
|
1664
|
+
getLogitsWeights(this.#state),
|
|
1665
|
+
getLogitsConfig(this.#state),
|
|
1666
|
+
this.#state.useGPU,
|
|
1667
|
+
this.#state.debugFlags,
|
|
1668
|
+
undefined,
|
|
1669
|
+
debugCheckBuffer,
|
|
1670
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
1671
|
+
{ lastPositionOnly: true }
|
|
1672
|
+
);
|
|
1673
|
+
|
|
1674
|
+
lastLogits = logits.length === logitsVocabSize
|
|
1675
|
+
? logits
|
|
1676
|
+
: extractLastPositionLogits(logits, numTokens, logitsVocabSize);
|
|
1677
|
+
releaseBuffer(currentHiddenBuffer);
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
this.#state.currentSeqLen = startPos + numTokens;
|
|
1681
|
+
|
|
1682
|
+
if (usedRecordedLogits) {
|
|
1683
|
+
if (logitsVocabSize < config.vocabSize) {
|
|
1684
|
+
const padded = new Float32Array(config.vocabSize);
|
|
1685
|
+
padded.set(lastLogits);
|
|
1686
|
+
padded.fill(-Infinity, logitsVocabSize);
|
|
1687
|
+
lastLogits = padded;
|
|
1688
|
+
}
|
|
1689
|
+
if (config.finalLogitSoftcapping != null) {
|
|
1690
|
+
applySoftcapping(lastLogits, config.finalLogitSoftcapping);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
if (opts.debug) {
|
|
1695
|
+
logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
if (opts.debug) {
|
|
1699
|
+
if (this.#state.kvCache?.hasGPUCache?.()) {
|
|
1700
|
+
log.debug('Pipeline', `KV cache active after prefill: seqLen=${this.#state.kvCache.getKeyCache(0)?.constructor.name ?? '?'}`);
|
|
1701
|
+
} else {
|
|
1702
|
+
log.warn('Pipeline', `KV cache NOT active after prefill! hasGPUCache=${this.#state.kvCache?.hasGPUCache?.()}`);
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
return lastLogits;
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
async _decodeStep(currentIds, opts) {
|
|
1711
|
+
const debugCheckBuffer = this.#state.debug
|
|
1712
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
1713
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
1714
|
+
: undefined;
|
|
1715
|
+
return decodeStep(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
async decodeStepLogits(currentIds, options = {}) {
|
|
1719
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1720
|
+
if (this.#state.isGenerating && options.__internalGenerate !== true) {
|
|
1721
|
+
throw new Error('Generation already in progress');
|
|
1722
|
+
}
|
|
1723
|
+
resetActiveExecutionPlan(this.#state);
|
|
1724
|
+
|
|
1725
|
+
validateCallTimeOptions(options);
|
|
1726
|
+
|
|
1727
|
+
const opts = this._resolveStepOptions(options);
|
|
1728
|
+
return this._decodeStepToLogits(currentIds, opts);
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
async advanceWithToken(tokenId, options = {}) {
|
|
1732
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1733
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1734
|
+
resetActiveExecutionPlan(this.#state);
|
|
1735
|
+
|
|
1736
|
+
validateCallTimeOptions(options);
|
|
1737
|
+
|
|
1738
|
+
const opts = this._resolveStepOptions(options);
|
|
1739
|
+
const debugCheckBuffer = this.#state.debug
|
|
1740
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
1741
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
1742
|
+
: undefined;
|
|
1743
|
+
|
|
1744
|
+
this._assertTokenIdInRange(tokenId, 'advanceWithToken');
|
|
1745
|
+
await advanceWithToken(this.#state, tokenId, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
async advanceWithTokenAndEmbedding(tokenId, options = {}) {
|
|
1749
|
+
if (!this.#state.isLoaded) throw new Error('Model not loaded');
|
|
1750
|
+
if (this.#state.isGenerating) throw new Error('Generation already in progress');
|
|
1751
|
+
resetActiveExecutionPlan(this.#state);
|
|
1752
|
+
|
|
1753
|
+
validateCallTimeOptions(options);
|
|
1754
|
+
|
|
1755
|
+
const opts = this._resolveStepOptions(options);
|
|
1756
|
+
const embeddingMode = resolveAdvanceEmbeddingMode(this.#state, options);
|
|
1757
|
+
const debugCheckBuffer = this.#state.debug
|
|
1758
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
1759
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
1760
|
+
: undefined;
|
|
1761
|
+
|
|
1762
|
+
this._assertTokenIdInRange(tokenId, 'advanceWithTokenAndEmbedding');
|
|
1763
|
+
return runAdvanceWithTokenAndEmbedding(
|
|
1764
|
+
this.#state,
|
|
1765
|
+
tokenId,
|
|
1766
|
+
opts,
|
|
1767
|
+
this._getDecodeHelpers(debugCheckBuffer),
|
|
1768
|
+
embeddingMode
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
async _generateNTokensGPU(startToken, N, currentIds, opts) {
|
|
1773
|
+
const debugCheckBuffer = this.#state.debug
|
|
1774
|
+
? (buffer, label, numTokens, expectedDim) =>
|
|
1775
|
+
debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
|
|
1776
|
+
: undefined;
|
|
1777
|
+
return generateNTokensGPU(this.#state, startToken, N, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
|
|
1778
|
+
}
|
|
1779
|
+
}
|