doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -3,6 +3,7 @@ import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
|
|
|
3
3
|
import { recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
|
|
4
4
|
import { recordRepPenalty } from '../../../gpu/kernels/rep-penalty.js';
|
|
5
5
|
import { recordCheckStop } from '../../../gpu/kernels/check-stop.js';
|
|
6
|
+
import { recordCheckHotVocabStop } from '../../../gpu/kernels/check-hot-vocab-stop.js';
|
|
6
7
|
import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
|
|
7
8
|
import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
|
|
8
9
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
@@ -26,6 +27,13 @@ import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-run
|
|
|
26
27
|
import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
|
|
27
28
|
import { hasLinearAttentionLayers } from './linear-attention.js';
|
|
28
29
|
import { hasConvLayers } from './layer.js';
|
|
30
|
+
import {
|
|
31
|
+
preparePerLayerInputs,
|
|
32
|
+
createPleBufferCache,
|
|
33
|
+
prefetchPerLayerRow,
|
|
34
|
+
hasRangeBackedPerLayerInputEmbeddings,
|
|
35
|
+
getPleHotVocabularyRuntime,
|
|
36
|
+
} from './per-layer-inputs.js';
|
|
29
37
|
|
|
30
38
|
const UNKNOWN_TOKEN_TEXT = '<unknown>';
|
|
31
39
|
|
|
@@ -71,6 +79,64 @@ function isOwnedDecodeBuffer(candidate, decodeHiddenBuffer, decodeAltBuffer) {
|
|
|
71
79
|
return candidate === decodeAltBuffer;
|
|
72
80
|
}
|
|
73
81
|
|
|
82
|
+
function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
|
|
83
|
+
if (!buffer) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
|
|
87
|
+
if (ownsBuffer) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
|
|
91
|
+
if (cachedPleBuffer) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (recorder) {
|
|
95
|
+
recorder.trackTemporaryBuffer(buffer);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
releaseBuffer(buffer);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function schedulePlePrefetchForToken(state, tokenId) {
|
|
102
|
+
if (state?.prefetchPleNextToken !== true) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
const config = state.modelConfig;
|
|
106
|
+
const pleHiddenSize = Number(config?.hiddenSizePerLayerInput ?? 0);
|
|
107
|
+
if (!Number.isFinite(pleHiddenSize) || pleHiddenSize <= 0) {
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const pleWeights = state.weights.get('per_layer_inputs');
|
|
111
|
+
if (!pleWeights?.embedTokensPerLayer) {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const perLayerInputsSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
|
|
115
|
+
state.plePrefetchPending = prefetchPerLayerRow(
|
|
116
|
+
tokenId,
|
|
117
|
+
pleWeights.embedTokensPerLayer,
|
|
118
|
+
config.numLayers * pleHiddenSize,
|
|
119
|
+
perLayerInputsSession
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function getReusableSampleReadbackBuffer(state, device, size) {
|
|
124
|
+
const existing = state.sampleReadbackBuffer;
|
|
125
|
+
if (existing && existing.size >= size) {
|
|
126
|
+
return existing;
|
|
127
|
+
}
|
|
128
|
+
if (existing) {
|
|
129
|
+
existing.destroy();
|
|
130
|
+
}
|
|
131
|
+
const buffer = device.createBuffer({
|
|
132
|
+
label: 'sample_staging_reuse',
|
|
133
|
+
size,
|
|
134
|
+
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
|
|
135
|
+
});
|
|
136
|
+
state.sampleReadbackBuffer = buffer;
|
|
137
|
+
return buffer;
|
|
138
|
+
}
|
|
139
|
+
|
|
74
140
|
export class FinitenessError extends Error {
|
|
75
141
|
constructor(message) {
|
|
76
142
|
super(message);
|
|
@@ -144,11 +210,15 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
144
210
|
const ownsStagingBuffer = options.ownsStagingBuffer === true;
|
|
145
211
|
const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
|
|
146
212
|
const ring = options.ring ?? null;
|
|
213
|
+
const cleanupRecorder = options.cleanupRecorder ?? null;
|
|
147
214
|
let mapped = false;
|
|
215
|
+
let cleanupCompleted = false;
|
|
148
216
|
|
|
149
217
|
try {
|
|
150
218
|
await stagingBuffer.mapAsync(GPUMapMode.READ);
|
|
151
219
|
mapped = true;
|
|
220
|
+
cleanupRecorder?.completeDeferredCleanup();
|
|
221
|
+
cleanupCompleted = true;
|
|
152
222
|
const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
|
|
153
223
|
return {
|
|
154
224
|
nextToken: mappedWords[0],
|
|
@@ -160,6 +230,9 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
160
230
|
if (mapped) {
|
|
161
231
|
stagingBuffer.unmap();
|
|
162
232
|
}
|
|
233
|
+
if (!cleanupCompleted) {
|
|
234
|
+
cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
235
|
+
}
|
|
163
236
|
if (ownsStagingBuffer) {
|
|
164
237
|
stagingBuffer.destroy();
|
|
165
238
|
}
|
|
@@ -167,6 +240,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
167
240
|
}
|
|
168
241
|
}
|
|
169
242
|
|
|
243
|
+
|
|
170
244
|
export async function readMappedBufferCopy(stagingBuffer, options = {}) {
|
|
171
245
|
const ownsStagingBuffer = options.ownsStagingBuffer !== false;
|
|
172
246
|
let mapped = false;
|
|
@@ -193,11 +267,14 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
193
267
|
tokenCount,
|
|
194
268
|
ownsTokensStaging = false,
|
|
195
269
|
ownsStopStaging = false,
|
|
270
|
+
ownsFinitenessStaging = Boolean(finitenessStagingBuffer),
|
|
196
271
|
ring = null,
|
|
272
|
+
cleanupRecorder = null,
|
|
197
273
|
} = options;
|
|
198
274
|
let tokensMapped = false;
|
|
199
275
|
let stopMapped = false;
|
|
200
276
|
let finitenessMapped = false;
|
|
277
|
+
let cleanupCompleted = false;
|
|
201
278
|
|
|
202
279
|
try {
|
|
203
280
|
const mapPromises = [tokensStagingBuffer.mapAsync(GPUMapMode.READ)];
|
|
@@ -207,10 +284,17 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
207
284
|
if (finitenessStagingBuffer) {
|
|
208
285
|
mapPromises.push(finitenessStagingBuffer.mapAsync(GPUMapMode.READ));
|
|
209
286
|
}
|
|
210
|
-
await Promise.
|
|
211
|
-
tokensMapped =
|
|
212
|
-
stopMapped = Boolean(stopStagingBuffer);
|
|
213
|
-
finitenessMapped = Boolean(finitenessStagingBuffer)
|
|
287
|
+
const mapResults = await Promise.allSettled(mapPromises);
|
|
288
|
+
tokensMapped = mapResults[0]?.status === 'fulfilled';
|
|
289
|
+
stopMapped = Boolean(stopStagingBuffer) && mapResults[1]?.status === 'fulfilled';
|
|
290
|
+
finitenessMapped = Boolean(finitenessStagingBuffer)
|
|
291
|
+
&& mapResults[stopStagingBuffer ? 2 : 1]?.status === 'fulfilled';
|
|
292
|
+
const mapFailure = mapResults.find((result) => result.status === 'rejected');
|
|
293
|
+
if (mapFailure) {
|
|
294
|
+
throw mapFailure.reason;
|
|
295
|
+
}
|
|
296
|
+
cleanupRecorder?.completeDeferredCleanup();
|
|
297
|
+
cleanupCompleted = true;
|
|
214
298
|
|
|
215
299
|
const tokens = Array.from(
|
|
216
300
|
new Uint32Array(tokensStagingBuffer.getMappedRange()).subarray(0, tokenCount)
|
|
@@ -237,7 +321,10 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
237
321
|
if (stopMapped) {
|
|
238
322
|
stopStagingBuffer.unmap();
|
|
239
323
|
}
|
|
240
|
-
if (
|
|
324
|
+
if (!cleanupCompleted) {
|
|
325
|
+
cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
326
|
+
}
|
|
327
|
+
if (ownsFinitenessStaging) {
|
|
241
328
|
finitenessStagingBuffer.destroy();
|
|
242
329
|
}
|
|
243
330
|
if (ownsTokensStaging) {
|
|
@@ -279,22 +366,53 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
|
|
|
279
366
|
outputBuffer: decodeHiddenBuffer ?? undefined,
|
|
280
367
|
transpose: state.embeddingTranspose,
|
|
281
368
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
369
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
282
370
|
activationDtype,
|
|
283
371
|
embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
|
|
372
|
+
executionPolicies: state.executionV1State?.policies ?? null,
|
|
284
373
|
});
|
|
285
374
|
|
|
286
375
|
let hiddenStates = embedTensor.buffer;
|
|
287
376
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
state.
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
377
|
+
// Resolve pending PLE prefetch from previous decode step
|
|
378
|
+
let plePrefetchResult = null;
|
|
379
|
+
if (state.plePrefetchPending) {
|
|
380
|
+
plePrefetchResult = await state.plePrefetchPending;
|
|
381
|
+
state.plePrefetchPending = null;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const perLayerInputs = await preparePerLayerInputs([tokenId], embedTensor, context, {
|
|
385
|
+
numTokens: 1,
|
|
386
|
+
pleCache: state.pleCache ?? null,
|
|
387
|
+
prefetchedRow: plePrefetchResult,
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
try {
|
|
391
|
+
for (let l = 0; l < config.numLayers; l++) {
|
|
392
|
+
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
393
|
+
const prevStates = hiddenStates;
|
|
394
|
+
hiddenStates = (await processLayer(l, hiddenStates, 1, false, context));
|
|
395
|
+
state.decodeBuffers.swapPingPong();
|
|
396
|
+
releasePerLayerInputBuffer(context.perLayerInputBuffer, null, context.decodeBuffers, state.pleCache ?? null);
|
|
397
|
+
if (perLayerInputs) {
|
|
398
|
+
perLayerInputs[l] = null;
|
|
399
|
+
}
|
|
400
|
+
context.perLayerInputBuffer = null;
|
|
401
|
+
if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
|
|
402
|
+
const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
|
|
403
|
+
if (!isPreAllocated) {
|
|
404
|
+
releaseBuffer(prevStates);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
} finally {
|
|
409
|
+
context.perLayerInputBuffer = null;
|
|
410
|
+
if (perLayerInputs) {
|
|
411
|
+
for (const buffer of perLayerInputs) {
|
|
412
|
+
releasePerLayerInputBuffer(buffer, null, context.decodeBuffers, state.pleCache ?? null);
|
|
296
413
|
}
|
|
297
414
|
}
|
|
415
|
+
helpers.releaseSharedAttentionState?.(context.sharedAttentionState, null);
|
|
298
416
|
}
|
|
299
417
|
|
|
300
418
|
return { hiddenStates, decodeHiddenBuffer, decodeAltBuffer, debugCheckBuffer, context };
|
|
@@ -399,12 +517,27 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
399
517
|
outputBuffer: decodeHiddenBuffer ?? undefined,
|
|
400
518
|
transpose: state.embeddingTranspose,
|
|
401
519
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
520
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
402
521
|
activationDtype,
|
|
403
522
|
embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
|
|
523
|
+
executionPolicies: state.executionV1State?.policies ?? null,
|
|
404
524
|
});
|
|
405
525
|
|
|
406
526
|
let hiddenStates = embedTensor.buffer;
|
|
407
527
|
|
|
528
|
+
// Resolve pending PLE prefetch from previous decode step
|
|
529
|
+
let plePrefetchResult = null;
|
|
530
|
+
if (state.plePrefetchPending) {
|
|
531
|
+
plePrefetchResult = await state.plePrefetchPending;
|
|
532
|
+
state.plePrefetchPending = null;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
const perLayerInputs = await preparePerLayerInputs([lastToken], embedTensor, context, {
|
|
536
|
+
numTokens: 1,
|
|
537
|
+
pleCache: state.pleCache ?? null,
|
|
538
|
+
prefetchedRow: plePrefetchResult,
|
|
539
|
+
});
|
|
540
|
+
|
|
408
541
|
if (opts.debug && state.decodeStepCount === 1) {
|
|
409
542
|
const validSize = config.hiddenSize * activationBytes;
|
|
410
543
|
const embedData = await readBuffer(hiddenStates, validSize);
|
|
@@ -426,22 +559,48 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
426
559
|
log.debug('Decode', `KV cache check: hasGPUCache=${hasGPUCache}, currentSeqLen=${context.currentSeqLen}`);
|
|
427
560
|
}
|
|
428
561
|
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
562
|
+
try {
|
|
563
|
+
for (let l = 0; l < config.numLayers; l++) {
|
|
564
|
+
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
565
|
+
const prevStates = hiddenStates;
|
|
566
|
+
hiddenStates = (await processLayer(l, hiddenStates, numTokens, false, context));
|
|
567
|
+
|
|
568
|
+
state.decodeBuffers.swapPingPong();
|
|
569
|
+
releasePerLayerInputBuffer(
|
|
570
|
+
context.perLayerInputBuffer,
|
|
571
|
+
recorder,
|
|
572
|
+
context.decodeBuffers,
|
|
573
|
+
state.pleCache ?? null
|
|
574
|
+
);
|
|
575
|
+
if (perLayerInputs) {
|
|
576
|
+
perLayerInputs[l] = null;
|
|
577
|
+
}
|
|
578
|
+
context.perLayerInputBuffer = null;
|
|
434
579
|
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
580
|
+
if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
|
|
581
|
+
const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
|
|
582
|
+
if (!isPreAllocated) {
|
|
583
|
+
if (recorder) {
|
|
584
|
+
recorder.trackTemporaryBuffer(prevStates);
|
|
585
|
+
} else {
|
|
586
|
+
releaseBuffer(prevStates);
|
|
587
|
+
}
|
|
442
588
|
}
|
|
443
589
|
}
|
|
444
590
|
}
|
|
591
|
+
} finally {
|
|
592
|
+
context.perLayerInputBuffer = null;
|
|
593
|
+
if (perLayerInputs) {
|
|
594
|
+
for (const buffer of perLayerInputs) {
|
|
595
|
+
releasePerLayerInputBuffer(
|
|
596
|
+
buffer,
|
|
597
|
+
recorder,
|
|
598
|
+
context.decodeBuffers,
|
|
599
|
+
state.pleCache ?? null
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
|
|
445
604
|
}
|
|
446
605
|
|
|
447
606
|
const logitSoftcap = config.finalLogitSoftcapping === null
|
|
@@ -478,6 +637,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
478
637
|
numTokens,
|
|
479
638
|
helpers.getLogitsWeights(),
|
|
480
639
|
helpers.getLogitsConfig(),
|
|
640
|
+
state.operatorDiagnostics,
|
|
481
641
|
);
|
|
482
642
|
|
|
483
643
|
const ringTokensBuffer = ringSlot?.tokens ?? null;
|
|
@@ -503,12 +663,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
503
663
|
|
|
504
664
|
const ringStagingBuffer = ringSlot?.stagingTokens ?? null;
|
|
505
665
|
const stagingSize = state.finitenessBuffer ? 20 : 4;
|
|
506
|
-
const stagingBuffer =
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
});
|
|
511
|
-
const ownsStagingBuffer = stagingBuffer !== ringStagingBuffer;
|
|
666
|
+
const stagingBuffer = ringStagingBuffer && ringStagingBuffer.size >= stagingSize
|
|
667
|
+
? ringStagingBuffer
|
|
668
|
+
: getReusableSampleReadbackBuffer(state, device, stagingSize);
|
|
669
|
+
const ownsStagingBuffer = false;
|
|
512
670
|
const ownsSampleOutputBuffer = !ringTokensBuffer || sampleOutputBuffer !== ringTokensBuffer;
|
|
513
671
|
|
|
514
672
|
const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
|
|
@@ -518,25 +676,37 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
518
676
|
encoder.copyBufferToBuffer(state.finitenessBuffer, 0, stagingBuffer, 4, 16);
|
|
519
677
|
}
|
|
520
678
|
|
|
521
|
-
const
|
|
522
|
-
|
|
523
|
-
|
|
679
|
+
const readbackMode = executionPlan?.readbackMode;
|
|
680
|
+
const isOverlapped = readbackMode === 'overlapped';
|
|
681
|
+
|
|
682
|
+
// In overlapped mode, advance ring BEFORE submit so the GPU's next copy
|
|
683
|
+
// target is a fresh slot while we read the current one.
|
|
684
|
+
if (isOverlapped) {
|
|
685
|
+
ring?.advance();
|
|
686
|
+
}
|
|
524
687
|
|
|
525
688
|
if (!allowReadback('pipeline.decode.sample')) {
|
|
526
689
|
throw new Error('[Pipeline] GPU readback disabled for sampling');
|
|
527
690
|
}
|
|
528
691
|
|
|
692
|
+
const submitStart = performance.now();
|
|
693
|
+
recorder.submit({ cleanup: 'deferred' });
|
|
694
|
+
const submitWaitMs = performance.now() - submitStart;
|
|
695
|
+
|
|
529
696
|
const readbackStart = performance.now();
|
|
530
|
-
const
|
|
697
|
+
const readbackResult = await readSampledTokenFromStagingBuffer(stagingBuffer, {
|
|
531
698
|
ownsStagingBuffer,
|
|
532
699
|
hasFinitenessBuffer: Boolean(state.finitenessBuffer),
|
|
533
|
-
ring,
|
|
700
|
+
ring: isOverlapped ? null : ring,
|
|
701
|
+
cleanupRecorder: recorder,
|
|
534
702
|
});
|
|
535
703
|
const readbackWaitMs = performance.now() - readbackStart;
|
|
536
704
|
|
|
537
705
|
state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
|
|
538
706
|
state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
|
|
539
707
|
|
|
708
|
+
const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
|
|
709
|
+
|
|
540
710
|
if (finitenessStatus.triggered) {
|
|
541
711
|
releaseBuffer(logitsBuffer);
|
|
542
712
|
if (ownsSampleOutputBuffer) releaseBuffer(sampleOutputBuffer);
|
|
@@ -544,13 +714,16 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
544
714
|
throw new FinitenessError(`F16 bounds exceeded during generation${finitenessStatus.metadata}`);
|
|
545
715
|
}
|
|
546
716
|
|
|
547
|
-
log.debug('Decode', `Step ${state.decodeStepCount}: token=${
|
|
717
|
+
log.debug('Decode', `Step ${state.decodeStepCount}: token=${fusedNextToken} (vocabSize=${config.vocabSize})`);
|
|
548
718
|
|
|
549
|
-
const invalidToken =
|
|
550
|
-
|| (padTokenId != null &&
|
|
551
|
-
|| (padTokenId == null &&
|
|
719
|
+
const invalidToken = fusedNextToken >= config.vocabSize
|
|
720
|
+
|| (padTokenId != null && fusedNextToken === padTokenId)
|
|
721
|
+
|| (padTokenId == null && fusedNextToken === 0);
|
|
722
|
+
if (!invalidToken) {
|
|
723
|
+
schedulePlePrefetchForToken(state, fusedNextToken);
|
|
724
|
+
}
|
|
552
725
|
if (invalidToken) {
|
|
553
|
-
log.warn('Decode', `Suspicious token ${
|
|
726
|
+
log.warn('Decode', `Suspicious token ${fusedNextToken} (vocabSize=${config.vocabSize}, step=${state.decodeStepCount})`);
|
|
554
727
|
if (allowReadback('pipeline.decode.debug-logits')) {
|
|
555
728
|
const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
|
|
556
729
|
const logitSample = await readBuffer(logitsBuffer, Math.min(config.vocabSize * logitsBytes, 4096));
|
|
@@ -610,7 +783,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
610
783
|
state.debugFlags,
|
|
611
784
|
undefined,
|
|
612
785
|
debugCheckBuffer,
|
|
613
|
-
state.runtimeConfig.shared.debug.probes
|
|
786
|
+
state.runtimeConfig.shared.debug.probes,
|
|
787
|
+
null,
|
|
788
|
+
state.operatorDiagnostics
|
|
614
789
|
);
|
|
615
790
|
applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty);
|
|
616
791
|
const fallbackToken = sample(fallbackLogits, {
|
|
@@ -620,6 +795,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
620
795
|
padTokenId,
|
|
621
796
|
seed: opts.seed,
|
|
622
797
|
});
|
|
798
|
+
schedulePlePrefetchForToken(state, fallbackToken);
|
|
623
799
|
if (!isPreAllocated) {
|
|
624
800
|
releaseBuffer(hiddenStates);
|
|
625
801
|
}
|
|
@@ -633,10 +809,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
633
809
|
|
|
634
810
|
state.currentSeqLen++;
|
|
635
811
|
const stepWallMs = performance.now() - stepWallStart;
|
|
636
|
-
const gpuMs = state.stats.gpuTimeDecodeMs ?? 0;
|
|
637
812
|
state.stats.singleTokenOrchestrationMs = (state.stats.singleTokenOrchestrationMs ?? 0)
|
|
638
813
|
+ Math.max(0, stepWallMs - submitWaitMs - readbackWaitMs);
|
|
639
|
-
return
|
|
814
|
+
return fusedNextToken;
|
|
640
815
|
}
|
|
641
816
|
|
|
642
817
|
await submitDecodeRecorderProfile(state, opts, recorder, ' (layers only)');
|
|
@@ -673,7 +848,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
673
848
|
numTokens,
|
|
674
849
|
helpers.getLogitsWeights(),
|
|
675
850
|
helpers.getLogitsConfig(),
|
|
676
|
-
state.debugFlags
|
|
851
|
+
state.debugFlags,
|
|
852
|
+
state.operatorDiagnostics
|
|
677
853
|
);
|
|
678
854
|
if (logitsResult) {
|
|
679
855
|
const { logitsBuffer, vocabSize, logitsDtype } = logitsResult;
|
|
@@ -691,7 +867,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
691
867
|
vocabSize,
|
|
692
868
|
config.vocabSize,
|
|
693
869
|
config,
|
|
694
|
-
state.runtimeConfig.shared.debug.probes
|
|
870
|
+
state.runtimeConfig.shared.debug.probes,
|
|
871
|
+
state.operatorDiagnostics
|
|
695
872
|
);
|
|
696
873
|
const sampledLogits = extractLastPositionLogits(finalizedLogits, numTokens, config.vocabSize);
|
|
697
874
|
|
|
@@ -736,7 +913,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
736
913
|
state.debugFlags,
|
|
737
914
|
undefined,
|
|
738
915
|
debugCheckBuffer,
|
|
739
|
-
state.runtimeConfig.shared.debug.probes
|
|
916
|
+
state.runtimeConfig.shared.debug.probes,
|
|
917
|
+
null,
|
|
918
|
+
state.operatorDiagnostics
|
|
740
919
|
);
|
|
741
920
|
|
|
742
921
|
if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
|
|
@@ -794,7 +973,8 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
|
|
|
794
973
|
numTokens,
|
|
795
974
|
helpers.getLogitsWeights(),
|
|
796
975
|
helpers.getLogitsConfig(),
|
|
797
|
-
state.debugFlags
|
|
976
|
+
state.debugFlags,
|
|
977
|
+
state.operatorDiagnostics
|
|
798
978
|
);
|
|
799
979
|
|
|
800
980
|
if (logitsResult) {
|
|
@@ -811,7 +991,8 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
|
|
|
811
991
|
rawVocabSize,
|
|
812
992
|
config.vocabSize,
|
|
813
993
|
config,
|
|
814
|
-
state.runtimeConfig.shared.debug.probes
|
|
994
|
+
state.runtimeConfig.shared.debug.probes,
|
|
995
|
+
state.operatorDiagnostics
|
|
815
996
|
);
|
|
816
997
|
logits = extractLastPositionLogits(finalized, numTokens, config.vocabSize);
|
|
817
998
|
}
|
|
@@ -827,7 +1008,9 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
|
|
|
827
1008
|
state.debugFlags,
|
|
828
1009
|
undefined,
|
|
829
1010
|
debugCheckBuffer,
|
|
830
|
-
state.runtimeConfig.shared.debug.probes
|
|
1011
|
+
state.runtimeConfig.shared.debug.probes,
|
|
1012
|
+
null,
|
|
1013
|
+
state.operatorDiagnostics
|
|
831
1014
|
);
|
|
832
1015
|
logits = extractLastPositionLogits(rawLogits, numTokens, config.vocabSize);
|
|
833
1016
|
}
|
|
@@ -940,6 +1123,61 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
|
|
|
940
1123
|
};
|
|
941
1124
|
}
|
|
942
1125
|
|
|
1126
|
+
async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, opts, helpers) {
|
|
1127
|
+
const config = state.modelConfig;
|
|
1128
|
+
const batchStart = performance.now();
|
|
1129
|
+
state.batchingStats.batchedForwardCalls += 1;
|
|
1130
|
+
|
|
1131
|
+
const stopTokenIds = config.stopTokenIds;
|
|
1132
|
+
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
1133
|
+
const pleHiddenSize = Number(config.hiddenSizePerLayerInput ?? 0);
|
|
1134
|
+
const totalPerLayerHiddenSize = pleHiddenSize > 0
|
|
1135
|
+
? config.numLayers * pleHiddenSize
|
|
1136
|
+
: 0;
|
|
1137
|
+
const pleWeights = state.weights.get('per_layer_inputs');
|
|
1138
|
+
const pleSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
|
|
1139
|
+
const generatedTokens = [];
|
|
1140
|
+
const rollingIds = Array.isArray(currentIds) ? currentIds.slice() : Array.from(currentIds ?? []);
|
|
1141
|
+
let gpuSubmissions = 0;
|
|
1142
|
+
|
|
1143
|
+
try {
|
|
1144
|
+
state.prefetchPleNextToken = true;
|
|
1145
|
+
if (
|
|
1146
|
+
totalPerLayerHiddenSize > 0
|
|
1147
|
+
&& pleWeights?.embedTokensPerLayer
|
|
1148
|
+
&& !state.plePrefetchPending
|
|
1149
|
+
&& rollingIds.length > 0
|
|
1150
|
+
) {
|
|
1151
|
+
state.plePrefetchPending = prefetchPerLayerRow(
|
|
1152
|
+
rollingIds[rollingIds.length - 1],
|
|
1153
|
+
pleWeights.embedTokensPerLayer,
|
|
1154
|
+
totalPerLayerHiddenSize,
|
|
1155
|
+
pleSession
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
|
|
1159
|
+
for (let i = 0; i < N; i += 1) {
|
|
1160
|
+
const nextToken = await decodeStep(state, rollingIds, opts, helpers);
|
|
1161
|
+
gpuSubmissions += 1;
|
|
1162
|
+
generatedTokens.push(nextToken);
|
|
1163
|
+
rollingIds.push(nextToken);
|
|
1164
|
+
|
|
1165
|
+
if (isStopToken(nextToken, stopTokenIds, eosToken)) {
|
|
1166
|
+
break;
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
return {
|
|
1171
|
+
tokens: generatedTokens,
|
|
1172
|
+
actualCount: generatedTokens.length,
|
|
1173
|
+
};
|
|
1174
|
+
} finally {
|
|
1175
|
+
state.prefetchPleNextToken = false;
|
|
1176
|
+
state.batchingStats.totalBatchedTimeMs += Math.max(0, performance.now() - batchStart);
|
|
1177
|
+
state.batchingStats.gpuSubmissions += gpuSubmissions;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
|
|
943
1181
|
export async function generateNTokensGPU(state, startToken, N, currentIds, opts, helpers) {
|
|
944
1182
|
const device = getDevice();
|
|
945
1183
|
const config = state.modelConfig;
|
|
@@ -957,8 +1195,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
957
1195
|
// GPU stop-flag checks are only useful when we read back every token.
|
|
958
1196
|
// With deferred readback, we already scan sampled tokens on CPU to find the
|
|
959
1197
|
// earliest stop token, so extra stop buffers/kernels are redundant overhead.
|
|
960
|
-
|
|
961
|
-
|
|
1198
|
+
let useGpuStopFlags = stopCheckMode === 'per-token' && readbackInterval <= 1;
|
|
1199
|
+
let effectiveStopCheckMode = useGpuStopFlags ? 'per-token' : 'batch';
|
|
962
1200
|
const batchStart = performance.now();
|
|
963
1201
|
|
|
964
1202
|
state.batchingStats.batchedForwardCalls += 1;
|
|
@@ -978,6 +1216,30 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
978
1216
|
throw new Error('[Pipeline] Batch size exceeds decode ring capacity.');
|
|
979
1217
|
}
|
|
980
1218
|
|
|
1219
|
+
const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
|
|
1220
|
+
config,
|
|
1221
|
+
weights: state.weights,
|
|
1222
|
+
});
|
|
1223
|
+
const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: state.weights });
|
|
1224
|
+
const hotStartTokenIndex = pleHotVocabularyRuntime?.hotTokenIndexMap?.[startToken] ?? null;
|
|
1225
|
+
const canUseHotVocabularyBatchDecode = hasRangeBackedPerLayerInputs
|
|
1226
|
+
&& pleHotVocabularyRuntime
|
|
1227
|
+
&& hotStartTokenIndex != null
|
|
1228
|
+
&& hotStartTokenIndex !== pleHotVocabularyRuntime.sentinelIndex;
|
|
1229
|
+
if (hasRangeBackedPerLayerInputs && !canUseHotVocabularyBatchDecode) {
|
|
1230
|
+
return generateNTokensGPUStepwiseRangeBackedPle(
|
|
1231
|
+
state,
|
|
1232
|
+
N,
|
|
1233
|
+
currentIds,
|
|
1234
|
+
opts,
|
|
1235
|
+
helpers
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
if (canUseHotVocabularyBatchDecode) {
|
|
1239
|
+
useGpuStopFlags = true;
|
|
1240
|
+
effectiveStopCheckMode = 'per-token';
|
|
1241
|
+
}
|
|
1242
|
+
|
|
981
1243
|
const stopTokenIds = config.stopTokenIds;
|
|
982
1244
|
const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
|
|
983
1245
|
const padTokenId = state.tokenizer?.getSpecialTokens?.()?.pad ?? null;
|
|
@@ -991,7 +1253,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
991
1253
|
if (eosTokenId == null) {
|
|
992
1254
|
throw new Error('[Pipeline] Missing EOS token. Ensure tokenizer or manifest provides stop tokens.');
|
|
993
1255
|
}
|
|
994
|
-
const maxTokens = executionPlan?.maxTokens
|
|
1256
|
+
const maxTokens = executionPlan?.maxTokens
|
|
1257
|
+
?? opts.maxTokens
|
|
1258
|
+
?? state.runtimeConfig.inference.generation.maxTokens;
|
|
995
1259
|
const maxSeqLen = state.currentSeqLen + maxTokens;
|
|
996
1260
|
|
|
997
1261
|
const recordStart = performance.now();
|
|
@@ -1025,6 +1289,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1025
1289
|
})
|
|
1026
1290
|
: null;
|
|
1027
1291
|
const ownsStopBuffer = useGpuStopFlags && !ringSlot?.stop;
|
|
1292
|
+
const pleInputTokensBuffer = canUseHotVocabularyBatchDecode
|
|
1293
|
+
? device.createBuffer({
|
|
1294
|
+
size: (tokenCapacity + 1) * 4,
|
|
1295
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
1296
|
+
label: 'ple_hot_input_tokens',
|
|
1297
|
+
})
|
|
1298
|
+
: null;
|
|
1028
1299
|
|
|
1029
1300
|
const tokensStagingBuffer = ringSlot?.stagingTokens ?? device.createBuffer({
|
|
1030
1301
|
size: N * 4,
|
|
@@ -1039,7 +1310,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1039
1310
|
})
|
|
1040
1311
|
: null;
|
|
1041
1312
|
const ownsStopStaging = useGpuStopFlags && !ringSlot?.stagingStop;
|
|
1042
|
-
|
|
1313
|
+
const finitenessStagingBuffer = state.finitenessBuffer
|
|
1314
|
+
? ringSlot?.stagingFiniteness ?? device.createBuffer({
|
|
1315
|
+
size: 16,
|
|
1316
|
+
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
1317
|
+
})
|
|
1318
|
+
: null;
|
|
1319
|
+
const ownsFinitenessStaging = Boolean(state.finitenessBuffer) && !ringSlot?.stagingFiniteness;
|
|
1043
1320
|
let readbackCleanupDelegated = false;
|
|
1044
1321
|
let repHistoryBuffer = null;
|
|
1045
1322
|
let repHistoryCount = 0;
|
|
@@ -1049,6 +1326,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1049
1326
|
}
|
|
1050
1327
|
|
|
1051
1328
|
device.queue.writeBuffer(tokensBuffer, 0, new Uint32Array([startToken]));
|
|
1329
|
+
if (pleInputTokensBuffer) {
|
|
1330
|
+
device.queue.writeBuffer(pleInputTokensBuffer, 0, new Uint32Array([hotStartTokenIndex]));
|
|
1331
|
+
}
|
|
1052
1332
|
if (stopBuffer) {
|
|
1053
1333
|
const stopElements = stopBuffer.size / 4;
|
|
1054
1334
|
const zeroStopData = ringSlot?.zeroStopData;
|
|
@@ -1085,10 +1365,15 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1085
1365
|
device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
|
|
1086
1366
|
}
|
|
1087
1367
|
|
|
1368
|
+
// Hoist loop-invariant values to avoid repeated rule lookups and allocations.
|
|
1369
|
+
const embeddingDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype });
|
|
1370
|
+
const debugProbes = state.runtimeConfig.shared.debug.probes;
|
|
1371
|
+
const currentTokenIdsArray = [startToken];
|
|
1372
|
+
|
|
1088
1373
|
for (let i = 0; i < N; i++) {
|
|
1089
1374
|
const currentPos = state.currentSeqLen + i;
|
|
1090
1375
|
context.currentSeqLen = currentPos;
|
|
1091
|
-
context.currentTokenIds =
|
|
1376
|
+
context.currentTokenIds = currentTokenIdsArray;
|
|
1092
1377
|
context.decodeBuffers?.resetPingPong();
|
|
1093
1378
|
|
|
1094
1379
|
const hiddenTensor = await embed(tokensBuffer, embedBuffer, {
|
|
@@ -1097,24 +1382,59 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1097
1382
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
1098
1383
|
recorder,
|
|
1099
1384
|
transpose: state.embeddingTranspose,
|
|
1100
|
-
debugProbes
|
|
1385
|
+
debugProbes,
|
|
1386
|
+
operatorDiagnostics: state.operatorDiagnostics,
|
|
1101
1387
|
activationDtype,
|
|
1102
|
-
embeddingDtype
|
|
1388
|
+
embeddingDtype,
|
|
1389
|
+
executionPolicies: state.executionV1State?.policies ?? null,
|
|
1103
1390
|
numTokens: 1,
|
|
1104
1391
|
indexOffset: i,
|
|
1105
1392
|
});
|
|
1106
1393
|
|
|
1107
1394
|
let hiddenStatesBuffer = hiddenTensor.buffer;
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1395
|
+
const perLayerInputs = await preparePerLayerInputs(tokensBuffer, hiddenTensor, context, {
|
|
1396
|
+
numTokens: 1,
|
|
1397
|
+
indexOffset: i,
|
|
1398
|
+
perLayerTokenIds: pleInputTokensBuffer,
|
|
1399
|
+
perLayerIndexOffset: i,
|
|
1400
|
+
pleCache: state.pleCache ?? null,
|
|
1401
|
+
});
|
|
1402
|
+
try {
|
|
1403
|
+
for (let l = 0; l < config.numLayers; l++) {
|
|
1404
|
+
context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
|
|
1405
|
+
const prevStates = hiddenStatesBuffer;
|
|
1406
|
+
hiddenStatesBuffer = (await processLayer(l, hiddenStatesBuffer, 1, false, context));
|
|
1407
|
+
context.decodeBuffers?.swapPingPong();
|
|
1408
|
+
releasePerLayerInputBuffer(
|
|
1409
|
+
context.perLayerInputBuffer,
|
|
1410
|
+
recorder,
|
|
1411
|
+
context.decodeBuffers,
|
|
1412
|
+
state.pleCache ?? null
|
|
1413
|
+
);
|
|
1414
|
+
if (perLayerInputs) {
|
|
1415
|
+
perLayerInputs[l] = null;
|
|
1416
|
+
}
|
|
1417
|
+
context.perLayerInputBuffer = null;
|
|
1418
|
+
if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStatesBuffer) {
|
|
1419
|
+
const ownsBuffer = context.decodeBuffers?.ownsBuffer(prevStates);
|
|
1420
|
+
if (!ownsBuffer) {
|
|
1421
|
+
recorder.trackTemporaryBuffer(prevStates);
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
} finally {
|
|
1426
|
+
context.perLayerInputBuffer = null;
|
|
1427
|
+
if (perLayerInputs) {
|
|
1428
|
+
for (const buffer of perLayerInputs) {
|
|
1429
|
+
releasePerLayerInputBuffer(
|
|
1430
|
+
buffer,
|
|
1431
|
+
recorder,
|
|
1432
|
+
context.decodeBuffers,
|
|
1433
|
+
state.pleCache ?? null
|
|
1434
|
+
);
|
|
1116
1435
|
}
|
|
1117
1436
|
}
|
|
1437
|
+
helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
|
|
1118
1438
|
}
|
|
1119
1439
|
|
|
1120
1440
|
const logits = await recordLogitsGPU(
|
|
@@ -1122,7 +1442,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1122
1442
|
hiddenStatesBuffer,
|
|
1123
1443
|
1,
|
|
1124
1444
|
helpers.getLogitsWeights(),
|
|
1125
|
-
helpers.getLogitsConfig()
|
|
1445
|
+
helpers.getLogitsConfig(),
|
|
1446
|
+
state.operatorDiagnostics
|
|
1126
1447
|
);
|
|
1127
1448
|
const { logitsBuffer, vocabSize, logitsDtype } = logits;
|
|
1128
1449
|
|
|
@@ -1160,7 +1481,19 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1160
1481
|
});
|
|
1161
1482
|
}
|
|
1162
1483
|
|
|
1163
|
-
const stopCheck =
|
|
1484
|
+
const stopCheck = canUseHotVocabularyBatchDecode
|
|
1485
|
+
? recordCheckHotVocabStop(recorder, {
|
|
1486
|
+
sampledTokenBuffer: tokensBuffer,
|
|
1487
|
+
nextInputTokenBuffer: pleInputTokensBuffer,
|
|
1488
|
+
hotTokenIndexMapBuffer: pleHotVocabularyRuntime.hotTokenIndexMapBuffer,
|
|
1489
|
+
hotTokenSentinel: pleHotVocabularyRuntime.sentinelIndex,
|
|
1490
|
+
shouldStopBuffer: stopBuffer,
|
|
1491
|
+
tokenIndex: outputIndex,
|
|
1492
|
+
eosTokenId,
|
|
1493
|
+
maxTokens: maxSeqLen,
|
|
1494
|
+
currentPos,
|
|
1495
|
+
})
|
|
1496
|
+
: useGpuStopFlags
|
|
1164
1497
|
? recordCheckStop(recorder, {
|
|
1165
1498
|
sampledTokenBuffer: tokensBuffer,
|
|
1166
1499
|
shouldStopBuffer: stopBuffer,
|
|
@@ -1191,20 +1524,16 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1191
1524
|
encoder.copyBufferToBuffer(stopBuffer, 4, stopStagingBuffer, 0, N * 4);
|
|
1192
1525
|
}
|
|
1193
1526
|
|
|
1194
|
-
if (state.finitenessBuffer) {
|
|
1195
|
-
finitenessStagingBuffer = device.createBuffer({
|
|
1196
|
-
size: 16,
|
|
1197
|
-
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ
|
|
1198
|
-
});
|
|
1527
|
+
if (state.finitenessBuffer && finitenessStagingBuffer) {
|
|
1199
1528
|
encoder.copyBufferToBuffer(state.finitenessBuffer, 0, finitenessStagingBuffer, 0, 16);
|
|
1200
1529
|
}
|
|
1201
1530
|
|
|
1202
|
-
recorder.submit();
|
|
1203
|
-
|
|
1204
1531
|
if (!allowReadback('pipeline.decode.sample')) {
|
|
1205
1532
|
throw new Error('[Pipeline] GPU readback disabled for sampling');
|
|
1206
1533
|
}
|
|
1207
1534
|
|
|
1535
|
+
recorder.submit({ cleanup: 'deferred' });
|
|
1536
|
+
|
|
1208
1537
|
const readbackStart = performance.now();
|
|
1209
1538
|
readbackCleanupDelegated = true;
|
|
1210
1539
|
const readback = await readBatchTokensFromStagingBuffers({
|
|
@@ -1214,7 +1543,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1214
1543
|
tokenCount: N,
|
|
1215
1544
|
ownsTokensStaging,
|
|
1216
1545
|
ownsStopStaging,
|
|
1546
|
+
ownsFinitenessStaging,
|
|
1217
1547
|
ring,
|
|
1548
|
+
cleanupRecorder: recorder,
|
|
1218
1549
|
});
|
|
1219
1550
|
const readbackWaitMs = performance.now() - readbackStart;
|
|
1220
1551
|
state.stats.decodeReadbackWaitMs = (state.stats.decodeReadbackWaitMs ?? 0) + readbackWaitMs;
|
|
@@ -1280,7 +1611,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1280
1611
|
state.batchingStats.gpuSubmissions += 1;
|
|
1281
1612
|
|
|
1282
1613
|
if (!readbackCleanupDelegated) {
|
|
1283
|
-
if (
|
|
1614
|
+
if (ownsFinitenessStaging) {
|
|
1284
1615
|
finitenessStagingBuffer.destroy();
|
|
1285
1616
|
}
|
|
1286
1617
|
if (ownsTokensStaging) tokensStagingBuffer.destroy();
|
|
@@ -1289,6 +1620,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1289
1620
|
}
|
|
1290
1621
|
if (ownsTokensBuffer) tokensBuffer.destroy();
|
|
1291
1622
|
if (ownsStopBuffer) stopBuffer?.destroy();
|
|
1623
|
+
pleInputTokensBuffer?.destroy();
|
|
1292
1624
|
if (repHistoryBuffer) repHistoryBuffer.destroy();
|
|
1293
1625
|
}
|
|
1294
1626
|
}
|