doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
2
|
+
import { READBACK_MODES } from '../../../config/schema/execution-v1.schema.js';
|
|
2
3
|
import {
|
|
3
4
|
resolveDeferredRoundingWindowTokens,
|
|
4
5
|
resolveRangeAwareSelectiveWideningConfig,
|
|
@@ -7,6 +8,7 @@ import { log } from '../../../debug/index.js';
|
|
|
7
8
|
|
|
8
9
|
export const PRIMARY_EXECUTION_PLAN_ID = 'primary';
|
|
9
10
|
export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
|
|
11
|
+
const DEFAULT_MAX_TOKENS = 256;
|
|
10
12
|
|
|
11
13
|
function assertOptionalBoolean(value, label) {
|
|
12
14
|
if (value === undefined) {
|
|
@@ -40,6 +42,15 @@ function assertOptionalStopCheckMode(value) {
|
|
|
40
42
|
return value;
|
|
41
43
|
}
|
|
42
44
|
|
|
45
|
+
function assertReadbackMode(value) {
|
|
46
|
+
if (!value || !READBACK_MODES.includes(value)) {
|
|
47
|
+
throw new Error(
|
|
48
|
+
`[ExecutionPlan] readbackMode must be one of ${READBACK_MODES.join(', ')}; got ${JSON.stringify(value)}.`
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
|
|
43
54
|
function resolveFallbackActivationDtype(primaryActivationDtype) {
|
|
44
55
|
const fallbackActivationDtype = selectRuleValue(
|
|
45
56
|
'inference',
|
|
@@ -55,6 +66,43 @@ function resolveFallbackActivationDtype(primaryActivationDtype) {
|
|
|
55
66
|
return fallbackActivationDtype;
|
|
56
67
|
}
|
|
57
68
|
|
|
69
|
+
function kernelUsesF16ActivationExecution(kernel) {
|
|
70
|
+
const kernelName = String(kernel ?? '');
|
|
71
|
+
if (!kernelName) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
return /_f16(?!w|kv)/.test(kernelName);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function stepUsesF16ActivationExecution(step) {
|
|
78
|
+
if (!step || typeof step !== 'object') {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
const precision = step.precision ?? null;
|
|
82
|
+
if (
|
|
83
|
+
precision?.activationDtype === 'f16'
|
|
84
|
+
|| precision?.inputDtype === 'f16'
|
|
85
|
+
|| precision?.outputDtype === 'f16'
|
|
86
|
+
) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
return kernelUsesF16ActivationExecution(step.kernel);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function kernelPathUsesF16ActivationExecution(kernelPath) {
|
|
93
|
+
const stepLists = [
|
|
94
|
+
kernelPath?.decode?.steps,
|
|
95
|
+
kernelPath?.prefill?.steps,
|
|
96
|
+
kernelPath?.preLayer,
|
|
97
|
+
kernelPath?.postLayer,
|
|
98
|
+
kernelPath?.sampling,
|
|
99
|
+
];
|
|
100
|
+
for (const override of kernelPath?.layerOverrides ?? []) {
|
|
101
|
+
stepLists.push(override?.steps, override?.decode?.steps, override?.prefill?.steps);
|
|
102
|
+
}
|
|
103
|
+
return stepLists.some((steps) => Array.isArray(steps) && steps.some((step) => stepUsesF16ActivationExecution(step)));
|
|
104
|
+
}
|
|
105
|
+
|
|
58
106
|
function createStaticExecutionPlan({
|
|
59
107
|
id,
|
|
60
108
|
source,
|
|
@@ -74,15 +122,17 @@ function createStaticExecutionPlan({
|
|
|
74
122
|
kernelPathSource,
|
|
75
123
|
activationDtype,
|
|
76
124
|
finitenessGuardEnabled: activationDtype === 'f16' && finitenessPolicy.enabled,
|
|
125
|
+
finitenessOnTrigger: finitenessPolicy.onTrigger,
|
|
77
126
|
finitenessAbsThreshold: finitenessPolicy.absThreshold,
|
|
78
127
|
finitenessIncludeNonFinite: finitenessPolicy.includeNonFinite,
|
|
79
128
|
deferredRoundingWindowTokens,
|
|
80
|
-
defaultDisableCommandBatching:
|
|
129
|
+
defaultDisableCommandBatching: batchingConfig.disableCommandBatching,
|
|
81
130
|
defaultDisableMultiTokenDecode: generationConfig.disableMultiTokenDecode,
|
|
82
131
|
defaultBatchSize: batchingConfig.batchSize,
|
|
83
132
|
defaultStopCheckMode: batchingConfig.stopCheckMode,
|
|
84
133
|
defaultMaxTokens: batchingConfig.maxTokens,
|
|
85
134
|
readbackInterval: batchingConfig.readbackInterval,
|
|
135
|
+
readbackMode: batchingConfig.readbackMode,
|
|
86
136
|
ringTokens: batchingConfig.ringTokens,
|
|
87
137
|
ringStop: batchingConfig.ringStop,
|
|
88
138
|
ringStaging: batchingConfig.ringStaging,
|
|
@@ -114,7 +164,8 @@ function validatePlanAgainstManifest(plan, manifest) {
|
|
|
114
164
|
const manifestInf = manifest.inference ?? manifest;
|
|
115
165
|
const warnings = [];
|
|
116
166
|
|
|
117
|
-
const manifestActivationDtype = manifestInf.compute?.activationDtype
|
|
167
|
+
const manifestActivationDtype = manifestInf.session?.compute?.defaults?.activationDtype
|
|
168
|
+
?? manifestInf.compute?.activationDtype
|
|
118
169
|
?? manifestInf.quantizationInfo?.compute
|
|
119
170
|
?? null;
|
|
120
171
|
if (manifestActivationDtype && plan.activationDtype
|
|
@@ -124,17 +175,6 @@ function validatePlanAgainstManifest(plan, manifest) {
|
|
|
124
175
|
);
|
|
125
176
|
}
|
|
126
177
|
|
|
127
|
-
const manifestKernelPathId = manifestInf.execution?.kernelPathId
|
|
128
|
-
?? manifestInf.defaultKernelPath
|
|
129
|
-
?? null;
|
|
130
|
-
if (manifestKernelPathId && plan.kernelPathId
|
|
131
|
-
&& plan.kernelPathId !== manifestKernelPathId
|
|
132
|
-
&& plan.kernelPathSource !== 'config') {
|
|
133
|
-
warnings.push(
|
|
134
|
-
`kernelPathId: plan="${plan.kernelPathId}" vs manifest="${manifestKernelPathId}"`
|
|
135
|
-
);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
178
|
if (warnings.length > 0) {
|
|
139
179
|
log.warn(
|
|
140
180
|
'ExecutionPlan',
|
|
@@ -157,11 +197,35 @@ export function compileExecutionPlanState(options) {
|
|
|
157
197
|
|
|
158
198
|
const inferenceConfig = runtimeConfig.inference;
|
|
159
199
|
const computeConfig = inferenceConfig.compute;
|
|
160
|
-
const generationConfig = inferenceConfig.generation;
|
|
161
|
-
|
|
200
|
+
const generationConfig = inferenceConfig.generation ?? {};
|
|
201
|
+
if (generationConfig.disableCommandBatching !== undefined) {
|
|
202
|
+
throw new Error(
|
|
203
|
+
'[ExecutionPlan] runtime.inference.generation.disableCommandBatching is removed. ' +
|
|
204
|
+
'Use runtime.inference.session.decodeLoop.disableCommandBatching.'
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
const sessionConfig = inferenceConfig.session ?? {};
|
|
208
|
+
const decodeLoopConfig = sessionConfig.decodeLoop;
|
|
209
|
+
if (!decodeLoopConfig || typeof decodeLoopConfig !== 'object') {
|
|
210
|
+
throw new Error('[ExecutionPlan] runtimeConfig.inference.session.decodeLoop is required.');
|
|
211
|
+
}
|
|
162
212
|
|
|
163
213
|
const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(computeConfig);
|
|
164
214
|
const deferredRoundingWindowTokens = resolveDeferredRoundingWindowTokens(computeConfig);
|
|
215
|
+
const batchingConfig = {
|
|
216
|
+
batchSize: decodeLoopConfig.batchSize,
|
|
217
|
+
stopCheckMode: decodeLoopConfig.stopCheckMode,
|
|
218
|
+
maxTokens: generationConfig.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
219
|
+
readbackInterval: decodeLoopConfig.readbackInterval,
|
|
220
|
+
readbackMode: assertReadbackMode(decodeLoopConfig.readbackMode),
|
|
221
|
+
ringTokens: decodeLoopConfig.ringTokens,
|
|
222
|
+
ringStop: decodeLoopConfig.ringStop,
|
|
223
|
+
ringStaging: decodeLoopConfig.ringStaging,
|
|
224
|
+
disableCommandBatching: assertOptionalBoolean(
|
|
225
|
+
decodeLoopConfig.disableCommandBatching,
|
|
226
|
+
'runtimeConfig.inference.session.decodeLoop.disableCommandBatching'
|
|
227
|
+
),
|
|
228
|
+
};
|
|
165
229
|
|
|
166
230
|
const primaryPlan = createStaticExecutionPlan({
|
|
167
231
|
id: PRIMARY_EXECUTION_PLAN_ID,
|
|
@@ -174,9 +238,11 @@ export function compileExecutionPlanState(options) {
|
|
|
174
238
|
generationConfig,
|
|
175
239
|
batchingConfig,
|
|
176
240
|
});
|
|
241
|
+
const primaryUsesF16ActivationExecution = primaryPlan.activationDtype === 'f16'
|
|
242
|
+
|| kernelPathUsesF16ActivationExecution(resolvedKernelPath);
|
|
177
243
|
|
|
178
244
|
let fallbackPlan = null;
|
|
179
|
-
if (primaryPlan.
|
|
245
|
+
if (primaryUsesF16ActivationExecution && primaryPlan.finitenessOnTrigger === 'fallback-plan') {
|
|
180
246
|
const fallbackActivationDtype = resolveFallbackActivationDtype(primaryPlan.activationDtype);
|
|
181
247
|
if (fallbackActivationDtype !== 'f32') {
|
|
182
248
|
throw new Error(
|
|
@@ -305,6 +371,10 @@ function resolveExecutionOverrides(options = {}) {
|
|
|
305
371
|
batchSize: assertOptionalPositiveInt(options.batchSize, 'batchSize'),
|
|
306
372
|
stopCheckMode: assertOptionalStopCheckMode(options.stopCheckMode),
|
|
307
373
|
maxTokens: assertOptionalPositiveInt(options.maxTokens, 'maxTokens'),
|
|
374
|
+
readbackInterval: assertOptionalPositiveInt(options.readbackInterval, 'readbackInterval'),
|
|
375
|
+
ringTokens: assertOptionalPositiveInt(options.ringTokens, 'ringTokens'),
|
|
376
|
+
ringStop: assertOptionalPositiveInt(options.ringStop, 'ringStop'),
|
|
377
|
+
ringStaging: assertOptionalPositiveInt(options.ringStaging, 'ringStaging'),
|
|
308
378
|
};
|
|
309
379
|
}
|
|
310
380
|
|
|
@@ -319,6 +389,7 @@ export function resolveExecutionSessionPlan(container, options = {}) {
|
|
|
319
389
|
kernelPathId: activePlan.kernelPathId,
|
|
320
390
|
activationDtype: activePlan.activationDtype,
|
|
321
391
|
finitenessGuardEnabled: activePlan.finitenessGuardEnabled,
|
|
392
|
+
finitenessOnTrigger: activePlan.finitenessOnTrigger,
|
|
322
393
|
finitenessAbsThreshold: activePlan.finitenessAbsThreshold,
|
|
323
394
|
finitenessIncludeNonFinite: activePlan.finitenessIncludeNonFinite,
|
|
324
395
|
deferredRoundingWindowTokens: activePlan.deferredRoundingWindowTokens,
|
|
@@ -327,10 +398,11 @@ export function resolveExecutionSessionPlan(container, options = {}) {
|
|
|
327
398
|
batchSize: overrides.batchSize ?? activePlan.defaultBatchSize,
|
|
328
399
|
stopCheckMode: overrides.stopCheckMode ?? activePlan.defaultStopCheckMode,
|
|
329
400
|
maxTokens: overrides.maxTokens ?? activePlan.defaultMaxTokens,
|
|
330
|
-
readbackInterval: activePlan.readbackInterval,
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
401
|
+
readbackInterval: overrides.readbackInterval ?? activePlan.readbackInterval,
|
|
402
|
+
readbackMode: activePlan.readbackMode,
|
|
403
|
+
ringTokens: overrides.ringTokens ?? activePlan.ringTokens,
|
|
404
|
+
ringStop: overrides.ringStop ?? activePlan.ringStop,
|
|
405
|
+
ringStaging: overrides.ringStaging ?? activePlan.ringStaging,
|
|
334
406
|
overrides,
|
|
335
407
|
};
|
|
336
408
|
}
|
|
@@ -349,9 +421,29 @@ export function isBatchDecodeEnabled(config) {
|
|
|
349
421
|
disableCommandBatching: config.disableCommandBatching,
|
|
350
422
|
isBdpaPagedLayout: config.isBdpaPagedLayout === true,
|
|
351
423
|
finitenessFallbackWindowOpen: config.finitenessFallbackWindowOpen === true,
|
|
424
|
+
hasLinearAttentionLayers: config.hasLinearAttentionLayers === true,
|
|
425
|
+
selfSpeculationEnabled: config.selfSpeculationEnabled === true,
|
|
426
|
+
hasRangeBackedPerLayerInputs: config.hasRangeBackedPerLayerInputs === true,
|
|
352
427
|
});
|
|
353
428
|
}
|
|
354
429
|
|
|
430
|
+
export function resolveMaxBatchDecodeTokens(config) {
|
|
431
|
+
const value = selectRuleValue('inference', 'execution', 'maxBatchDecodeTokens', {
|
|
432
|
+
hasHotVocabularyBatchDecode: config.hasHotVocabularyBatchDecode === true,
|
|
433
|
+
hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
|
|
434
|
+
hasLinearAttentionLayers: config.hasLinearAttentionLayers === true,
|
|
435
|
+
});
|
|
436
|
+
if (value == null) {
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
440
|
+
throw new Error(
|
|
441
|
+
`[ExecutionPlan] maxBatchDecodeTokens must be null or a positive integer; got ${JSON.stringify(value)}.`
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
return value;
|
|
445
|
+
}
|
|
446
|
+
|
|
355
447
|
export function isDecodeRecorderEnabled(config) {
|
|
356
448
|
return selectRuleValue('inference', 'execution', 'decodeRecorderEnabled', {
|
|
357
449
|
hasDevice: config.hasDevice === true,
|
|
@@ -4,7 +4,7 @@ export declare function normalizeDtype(value: unknown, label: string): 'f16' | '
|
|
|
4
4
|
export declare function isPhaseMatch(phase: string, targetPhase: string): boolean;
|
|
5
5
|
export declare function stepHasLayer(step: { layers: 'all' | number[] }, layerIdx: number): boolean;
|
|
6
6
|
export declare function requireSessionActivationDtype(
|
|
7
|
-
|
|
7
|
+
session: Record<string, unknown> | null | undefined,
|
|
8
8
|
label?: string
|
|
9
9
|
): 'f16' | 'f32';
|
|
10
10
|
|
|
@@ -14,7 +14,7 @@ export declare function resolveFinitenessFallbackKernelPathId(
|
|
|
14
14
|
|
|
15
15
|
export declare function buildInlineKernelPath(
|
|
16
16
|
steps: readonly Record<string, unknown>[],
|
|
17
|
-
|
|
17
|
+
session: Record<string, unknown> | null,
|
|
18
18
|
modelId: string,
|
|
19
19
|
numLayers: number,
|
|
20
20
|
finitenessFallbackKernelPathId?: string | null
|
|
@@ -22,12 +22,16 @@ export declare function buildInlineKernelPath(
|
|
|
22
22
|
|
|
23
23
|
export declare function assertKernelPathSessionCompatibility(
|
|
24
24
|
path: Record<string, unknown> | null | undefined,
|
|
25
|
-
|
|
25
|
+
session: Record<string, unknown> | null | undefined
|
|
26
26
|
): void;
|
|
27
27
|
|
|
28
28
|
export interface BuildLayerPipelineOptions {
|
|
29
29
|
/** When true, throws on incompatible ops instead of returning a degraded result. */
|
|
30
30
|
strict?: boolean;
|
|
31
|
+
/** When false, suppresses the degraded-pipeline log because another execution path remains active. */
|
|
32
|
+
logIncompatibleOps?: boolean;
|
|
33
|
+
/** When set, fused `ffn` steps inherit this dtype when they do not declare per-step precision. */
|
|
34
|
+
ffnDtypeFallback?: 'f16' | 'f32' | null;
|
|
31
35
|
}
|
|
32
36
|
|
|
33
37
|
export declare function buildLayerPipelineFromExecution(
|
|
@@ -38,7 +42,7 @@ export declare function buildLayerPipelineFromExecution(
|
|
|
38
42
|
| null;
|
|
39
43
|
|
|
40
44
|
export declare function buildSessionRuntimePatch(
|
|
41
|
-
|
|
45
|
+
session: Record<string, unknown> | null | undefined,
|
|
42
46
|
options?: {
|
|
43
47
|
includeDecodeLoop?: boolean;
|
|
44
48
|
}
|
|
@@ -36,10 +36,10 @@ export function stepHasLayer(step, layerIdx) {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export function requireSessionActivationDtype(
|
|
39
|
-
|
|
40
|
-
label = '
|
|
39
|
+
session,
|
|
40
|
+
label = 'session.compute.defaults.activationDtype'
|
|
41
41
|
) {
|
|
42
|
-
const activationDtype =
|
|
42
|
+
const activationDtype = session?.compute?.defaults?.activationDtype;
|
|
43
43
|
if (activationDtype == null) {
|
|
44
44
|
throw new Error(`[Execution] ${label} is required.`);
|
|
45
45
|
}
|
|
@@ -66,6 +66,22 @@ function toKernelPathStep(step) {
|
|
|
66
66
|
};
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
function applyFusedFfnKernelPathPrecision(step, activationDtype) {
|
|
70
|
+
if (!step || step.op !== 'ffn') {
|
|
71
|
+
return step;
|
|
72
|
+
}
|
|
73
|
+
const inputDtype = step.precision?.inputDtype ?? activationDtype;
|
|
74
|
+
const outputDtype = step.precision?.outputDtype ?? activationDtype;
|
|
75
|
+
return {
|
|
76
|
+
...step,
|
|
77
|
+
precision: {
|
|
78
|
+
...(step.precision ?? {}),
|
|
79
|
+
inputDtype,
|
|
80
|
+
outputDtype,
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
69
85
|
function getSectionSteps(steps, section, phase = null) {
|
|
70
86
|
return steps
|
|
71
87
|
.filter((step) => step.section === section)
|
|
@@ -89,20 +105,24 @@ function getInlineKernelPathSteps(path) {
|
|
|
89
105
|
...(path?.prefill?.steps ?? []),
|
|
90
106
|
...(path?.postLayer ?? []),
|
|
91
107
|
...(path?.sampling ?? []),
|
|
92
|
-
...(path?.layerOverrides?.flatMap((override) =>
|
|
108
|
+
...(path?.layerOverrides?.flatMap((override) => [
|
|
109
|
+
...(override?.steps ?? []),
|
|
110
|
+
...(override?.decode?.steps ?? []),
|
|
111
|
+
...(override?.prefill?.steps ?? []),
|
|
112
|
+
]) ?? []),
|
|
93
113
|
];
|
|
94
114
|
}
|
|
95
115
|
|
|
96
|
-
export function assertKernelPathSessionCompatibility(path,
|
|
116
|
+
export function assertKernelPathSessionCompatibility(path, session) {
|
|
97
117
|
if (!path) {
|
|
98
118
|
return;
|
|
99
119
|
}
|
|
100
120
|
const globalActivationDtype = normalizeDtype(
|
|
101
|
-
path.activationDtype ?? requireSessionActivationDtype(
|
|
121
|
+
path.activationDtype ?? requireSessionActivationDtype(session),
|
|
102
122
|
'inlineKernelPath.activationDtype'
|
|
103
123
|
);
|
|
104
124
|
const kvDtype = normalizeDtype(
|
|
105
|
-
path.kvDtype ??
|
|
125
|
+
path.kvDtype ?? session?.kvcache?.kvDtype ?? globalActivationDtype,
|
|
106
126
|
'inlineKernelPath.kvDtype'
|
|
107
127
|
);
|
|
108
128
|
|
|
@@ -161,15 +181,15 @@ export function resolveFinitenessFallbackKernelPathId(kernelPathId) {
|
|
|
161
181
|
|
|
162
182
|
export function buildInlineKernelPath(
|
|
163
183
|
steps,
|
|
164
|
-
|
|
184
|
+
session,
|
|
165
185
|
modelId,
|
|
166
186
|
numLayers,
|
|
167
187
|
finitenessFallbackKernelPathId = null
|
|
168
188
|
) {
|
|
169
|
-
const activationDtype = requireSessionActivationDtype(
|
|
189
|
+
const activationDtype = requireSessionActivationDtype(session);
|
|
170
190
|
const kvDtype = normalizeDtype(
|
|
171
|
-
|
|
172
|
-
'
|
|
191
|
+
session?.kvcache?.kvDtype ?? activationDtype,
|
|
192
|
+
'session.kvcache.kvDtype'
|
|
173
193
|
);
|
|
174
194
|
const decodeSteps = buildLayerPhaseSteps(steps, 'decode', 0);
|
|
175
195
|
const prefillSteps = buildLayerPhaseSteps(steps, 'prefill', 0);
|
|
@@ -193,24 +213,35 @@ export function buildInlineKernelPath(
|
|
|
193
213
|
steps: prefillSteps.length > 0 ? prefillSteps : decodeSteps,
|
|
194
214
|
},
|
|
195
215
|
};
|
|
216
|
+
path.decode.steps = path.decode.steps.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
|
|
217
|
+
path.prefill.steps = path.prefill.steps.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
|
|
196
218
|
|
|
197
219
|
if (numLayers > 0) {
|
|
198
220
|
const overrides = [];
|
|
199
221
|
for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
|
|
200
|
-
const decodeLayerSteps = buildLayerPhaseSteps(steps, 'decode', layerIdx)
|
|
201
|
-
|
|
222
|
+
const decodeLayerSteps = buildLayerPhaseSteps(steps, 'decode', layerIdx)
|
|
223
|
+
.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
|
|
224
|
+
const prefillLayerSteps = buildLayerPhaseSteps(steps, 'prefill', layerIdx)
|
|
225
|
+
.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
|
|
202
226
|
const hasCustomDecode = JSON.stringify(decodeLayerSteps) !== JSON.stringify(path.decode.steps);
|
|
203
227
|
const hasCustomPrefill = JSON.stringify(prefillLayerSteps) !== JSON.stringify(path.prefill.steps);
|
|
204
228
|
if (!hasCustomDecode && !hasCustomPrefill) continue;
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
229
|
+
const override = { layers: [layerIdx] };
|
|
230
|
+
if (
|
|
231
|
+
hasCustomDecode
|
|
232
|
+
&& hasCustomPrefill
|
|
233
|
+
&& JSON.stringify(decodeLayerSteps) === JSON.stringify(prefillLayerSteps)
|
|
234
|
+
) {
|
|
235
|
+
override.steps = decodeLayerSteps;
|
|
236
|
+
} else {
|
|
237
|
+
if (hasCustomDecode && decodeLayerSteps.length > 0) {
|
|
238
|
+
override.decode = { steps: decodeLayerSteps };
|
|
239
|
+
}
|
|
240
|
+
if (hasCustomPrefill && prefillLayerSteps.length > 0) {
|
|
241
|
+
override.prefill = { steps: prefillLayerSteps };
|
|
242
|
+
}
|
|
213
243
|
}
|
|
244
|
+
overrides.push(override);
|
|
214
245
|
}
|
|
215
246
|
if (overrides.length > 0) {
|
|
216
247
|
path.layerOverrides = overrides;
|
|
@@ -230,7 +261,7 @@ export function buildInlineKernelPath(
|
|
|
230
261
|
path.sampling = sampling;
|
|
231
262
|
}
|
|
232
263
|
|
|
233
|
-
assertKernelPathSessionCompatibility(path,
|
|
264
|
+
assertKernelPathSessionCompatibility(path, session);
|
|
234
265
|
return path;
|
|
235
266
|
}
|
|
236
267
|
|
|
@@ -244,7 +275,10 @@ export function buildInlineKernelPath(
|
|
|
244
275
|
* | { incompatibleOps: string[]; hasIncompatibleOps: true } | null}
|
|
245
276
|
*/
|
|
246
277
|
export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
247
|
-
const { strict = false } = options;
|
|
278
|
+
const { strict = false, logIncompatibleOps = true } = options;
|
|
279
|
+
const ffnDtypeFallback = options.ffnDtypeFallback == null
|
|
280
|
+
? null
|
|
281
|
+
: normalizeDtype(options.ffnDtypeFallback, 'buildLayerPipelineFromExecution.options.ffnDtypeFallback');
|
|
248
282
|
const layerSectionSteps = steps.filter((step) => step.section === 'layer');
|
|
249
283
|
if (layerSectionSteps.length === 0) {
|
|
250
284
|
return null;
|
|
@@ -257,13 +291,21 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
257
291
|
),
|
|
258
292
|
];
|
|
259
293
|
if (incompatibleOps.length > 0) {
|
|
260
|
-
const
|
|
294
|
+
const degradedMessage =
|
|
261
295
|
`[Execution] Layer pipeline contains ops not in PIPELINE_COMPATIBLE_OPS: ` +
|
|
262
296
|
`${incompatibleOps.join(', ')}. Pipeline will be degraded.`;
|
|
263
297
|
if (strict) {
|
|
264
|
-
throw new Error(
|
|
298
|
+
throw new Error(degradedMessage);
|
|
299
|
+
}
|
|
300
|
+
if (logIncompatibleOps) {
|
|
301
|
+
log.error('ExecutionRuntime', degradedMessage);
|
|
302
|
+
} else {
|
|
303
|
+
log.debug(
|
|
304
|
+
'ExecutionRuntime',
|
|
305
|
+
`[Execution] Layer pipeline contains ops not in PIPELINE_COMPATIBLE_OPS: ` +
|
|
306
|
+
`${incompatibleOps.join(', ')}. Inline kernel path remains active; JS layer pipeline disabled.`
|
|
307
|
+
);
|
|
265
308
|
}
|
|
266
|
-
log.error('ExecutionRuntime', message);
|
|
267
309
|
return { incompatibleOps, hasIncompatibleOps: true };
|
|
268
310
|
}
|
|
269
311
|
|
|
@@ -278,8 +320,13 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
278
320
|
...(step.b !== undefined ? { b: step.b } : {}),
|
|
279
321
|
...(step.variant !== undefined ? { variant: step.variant } : {}),
|
|
280
322
|
...(step.skipInputNorm !== undefined ? { skipInputNorm: step.skipInputNorm } : {}),
|
|
281
|
-
...(step.precision?.inputDtype
|
|
282
|
-
|
|
323
|
+
...(step.precision?.inputDtype
|
|
324
|
+
? { inputDtype: step.precision.inputDtype }
|
|
325
|
+
: (step.op === 'ffn' && ffnDtypeFallback ? { inputDtype: ffnDtypeFallback } : {})),
|
|
326
|
+
...(step.precision?.outputDtype
|
|
327
|
+
? { outputDtype: step.precision.outputDtype }
|
|
328
|
+
: (step.op === 'ffn' && ffnDtypeFallback ? { outputDtype: ffnDtypeFallback } : {})),
|
|
329
|
+
...(step.precision?.kvDtype ? { kvDtype: step.precision.kvDtype } : {}),
|
|
283
330
|
...(step.fromDtype ? { fromDtype: step.fromDtype } : {}),
|
|
284
331
|
...(step.toDtype ? { toDtype: step.toDtype } : {}),
|
|
285
332
|
...(step.probeStage ? { probeStage: step.probeStage } : {}),
|
|
@@ -295,7 +342,7 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
295
342
|
}
|
|
296
343
|
|
|
297
344
|
/**
|
|
298
|
-
* Build a runtime config patch from manifest
|
|
345
|
+
* Build a runtime config patch from manifest/runtime session.
|
|
299
346
|
*
|
|
300
347
|
* Field consumption status after merge into runtimeConfig.inference:
|
|
301
348
|
*
|
|
@@ -304,10 +351,9 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
304
351
|
* logits fallback (getRuntimeConfig().inference.compute.activationDtype),
|
|
305
352
|
* and layer context builder.
|
|
306
353
|
*
|
|
307
|
-
* CONSUMED (read by KV cache and
|
|
308
|
-
* - patch.kvcache.*
|
|
354
|
+
* CONSUMED (read by KV cache, batching, and execution-plan subsystems):
|
|
355
|
+
* - patch.session.kvcache.*
|
|
309
356
|
* - patch.batching.*
|
|
310
|
-
* - patch.generation.disableCommandBatching
|
|
311
357
|
*
|
|
312
358
|
* DEAD / NOT CONSUMED at runtime (merged into runtimeConfig but never read back):
|
|
313
359
|
* - patch.session.compute.defaults.mathDtype
|
|
@@ -318,10 +364,10 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
|
|
|
318
364
|
* future consumption. They should NOT be removed (non-breaking), but new code
|
|
319
365
|
* should not rely on reading them from runtimeConfig.inference.session.
|
|
320
366
|
*/
|
|
321
|
-
export function buildSessionRuntimePatch(
|
|
367
|
+
export function buildSessionRuntimePatch(session, options = {}) {
|
|
322
368
|
const includeDecodeLoop = options.includeDecodeLoop !== false;
|
|
323
369
|
const patch = {};
|
|
324
|
-
const computeDefaults =
|
|
370
|
+
const computeDefaults = session?.compute?.defaults ?? null;
|
|
325
371
|
const computePatch = {};
|
|
326
372
|
const sessionComputeDefaultsPatch = {};
|
|
327
373
|
const activationDtype = computeDefaults?.activationDtype;
|
|
@@ -360,23 +406,18 @@ export function buildSessionRuntimePatch(sessionDefaults, options = {}) {
|
|
|
360
406
|
},
|
|
361
407
|
};
|
|
362
408
|
}
|
|
363
|
-
if (
|
|
364
|
-
patch.kvcache = sessionDefaults.kvcache;
|
|
365
|
-
}
|
|
366
|
-
if (includeDecodeLoop && sessionDefaults?.decodeLoop) {
|
|
409
|
+
if (includeDecodeLoop && session?.decodeLoop) {
|
|
367
410
|
patch.batching = {
|
|
368
|
-
batchSize:
|
|
369
|
-
stopCheckMode:
|
|
370
|
-
readbackInterval:
|
|
371
|
-
ringTokens:
|
|
372
|
-
ringStop:
|
|
373
|
-
ringStaging:
|
|
411
|
+
batchSize: session.decodeLoop.batchSize,
|
|
412
|
+
stopCheckMode: session.decodeLoop.stopCheckMode,
|
|
413
|
+
readbackInterval: session.decodeLoop.readbackInterval,
|
|
414
|
+
ringTokens: session.decodeLoop.ringTokens,
|
|
415
|
+
ringStop: session.decodeLoop.ringStop,
|
|
416
|
+
ringStaging: session.decodeLoop.ringStaging,
|
|
374
417
|
};
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
};
|
|
379
|
-
}
|
|
418
|
+
}
|
|
419
|
+
if (session) {
|
|
420
|
+
patch.session = session;
|
|
380
421
|
}
|
|
381
422
|
return patch;
|
|
382
423
|
}
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
ExecutionV1GraphSchema,
|
|
3
|
-
|
|
3
|
+
ExecutionV1SessionSchema,
|
|
4
4
|
ExecutionV1PoliciesSchema,
|
|
5
5
|
ExecutionV1ExpandedStepSchema,
|
|
6
6
|
} from '../../../config/schema/execution-v1.schema.js';
|
|
7
|
+
import type { KernelPathSchema } from '../../../config/schema/index.js';
|
|
7
8
|
|
|
8
9
|
export interface ExecutionV1CompiledState {
|
|
9
|
-
|
|
10
|
+
session: ExecutionV1SessionSchema;
|
|
10
11
|
policies: ExecutionV1PoliciesSchema;
|
|
11
12
|
resolvedSteps: {
|
|
12
13
|
prefill: ExecutionV1ExpandedStepSchema[];
|
|
@@ -14,6 +15,8 @@ export interface ExecutionV1CompiledState {
|
|
|
14
15
|
all: ExecutionV1ExpandedStepSchema[];
|
|
15
16
|
};
|
|
16
17
|
runtimeInferencePatch: Record<string, unknown>;
|
|
18
|
+
appliedTransforms?: string[];
|
|
19
|
+
fallbackKernelPath?: KernelPathSchema | null;
|
|
17
20
|
}
|
|
18
21
|
|
|
19
22
|
export declare function hasExecutionV1(
|
|
@@ -24,16 +27,32 @@ export declare function compileExecutionV1(options?: {
|
|
|
24
27
|
manifestInference: {
|
|
25
28
|
schema: string;
|
|
26
29
|
execution: ExecutionV1GraphSchema;
|
|
27
|
-
|
|
30
|
+
session: ExecutionV1SessionSchema;
|
|
28
31
|
};
|
|
29
32
|
modelId?: string;
|
|
30
33
|
numLayers?: number;
|
|
34
|
+
runtimeSession?: ExecutionV1SessionSchema | null;
|
|
35
|
+
runtimeCompute?: Record<string, unknown> | null;
|
|
36
|
+
kernelPathPolicy?: Record<string, unknown> | null;
|
|
37
|
+
capabilities?: {
|
|
38
|
+
hasSubgroups?: boolean;
|
|
39
|
+
hasF16?: boolean;
|
|
40
|
+
hasSubgroupsF16?: boolean;
|
|
41
|
+
maxWorkgroupSize?: number;
|
|
42
|
+
maxBufferSize?: number;
|
|
43
|
+
} | null;
|
|
44
|
+
platform?: {
|
|
45
|
+
id?: string;
|
|
46
|
+
vendor?: string;
|
|
47
|
+
architecture?: string;
|
|
48
|
+
} | null;
|
|
31
49
|
}): ExecutionV1CompiledState;
|
|
32
50
|
|
|
33
51
|
export declare function applyExecutionV1RuntimeConfig(options?: {
|
|
34
52
|
runtimeConfig: Record<string, unknown>;
|
|
53
|
+
runtimeOverrides?: Record<string, unknown> | null;
|
|
35
54
|
manifest: {
|
|
36
|
-
inference?: { schema?: string; execution?: ExecutionV1GraphSchema;
|
|
55
|
+
inference?: { schema?: string; execution?: ExecutionV1GraphSchema; session?: ExecutionV1SessionSchema };
|
|
37
56
|
modelId?: string;
|
|
38
57
|
architecture?: { numLayers?: number };
|
|
39
58
|
};
|