doppler-gpu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
package/src/config/runtime.js
CHANGED
|
@@ -58,12 +58,19 @@ function assertNoDeprecatedRuntimeKeys(overrides) {
|
|
|
58
58
|
if (inference?.debug !== undefined) {
|
|
59
59
|
throw new Error('runtime.inference.debug is removed; use runtime.shared.debug');
|
|
60
60
|
}
|
|
61
|
-
// Deprecated in v0.9 — sampling
|
|
61
|
+
// Deprecated in v0.9 — sampling.maxTokens replaced by generation.maxTokens
|
|
62
62
|
if (inference?.sampling?.maxTokens !== undefined) {
|
|
63
|
-
throw new Error('sampling.maxTokens is removed; use inference.
|
|
63
|
+
throw new Error('sampling.maxTokens is removed; use inference.generation.maxTokens');
|
|
64
64
|
}
|
|
65
|
-
// Deprecated in v0.9 — session.maxNewTokens replaced by inference.
|
|
65
|
+
// Deprecated in v0.9 — session.maxNewTokens replaced by inference.generation.maxTokens
|
|
66
66
|
if (inference?.session?.maxNewTokens !== undefined) {
|
|
67
|
-
throw new Error('inference.session.maxNewTokens is not a supported runtime config key; use inference.
|
|
67
|
+
throw new Error('inference.session.maxNewTokens is not a supported runtime config key; use inference.generation.maxTokens');
|
|
68
|
+
}
|
|
69
|
+
// Deprecated in v0.9 — command batching policy now lives in the session decode loop
|
|
70
|
+
if (inference?.generation?.disableCommandBatching !== undefined) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
'inference.generation.disableCommandBatching is removed; ' +
|
|
73
|
+
'use inference.session.decodeLoop.disableCommandBatching'
|
|
74
|
+
);
|
|
68
75
|
}
|
|
69
76
|
}
|
|
@@ -14,7 +14,7 @@ import type {
|
|
|
14
14
|
} from './manifest.schema.js';
|
|
15
15
|
import type {
|
|
16
16
|
ExecutionV1GraphSchema,
|
|
17
|
-
|
|
17
|
+
ExecutionV1SessionSchema,
|
|
18
18
|
} from './execution-v1.schema.js';
|
|
19
19
|
|
|
20
20
|
export type ComputePrecision = 'f16' | 'f32' | 'auto' | null;
|
|
@@ -56,10 +56,12 @@ export interface ConverterManifestConfigSchema {
|
|
|
56
56
|
hashAlgorithm: HashAlgorithm;
|
|
57
57
|
optimizations: RuntimeOptimizationsSchema | null;
|
|
58
58
|
conversion: ConversionInfoSchema | null;
|
|
59
|
+
visionConfig?: Record<string, unknown> | null;
|
|
60
|
+
audioConfig?: Record<string, unknown> | null;
|
|
59
61
|
}
|
|
60
62
|
|
|
61
63
|
export interface ConverterInferenceConfigSchema {
|
|
62
|
-
|
|
64
|
+
session: ExecutionV1SessionSchema | null;
|
|
63
65
|
execution: ExecutionV1GraphSchema | null;
|
|
64
66
|
}
|
|
65
67
|
|
|
@@ -41,10 +41,12 @@ export const DEFAULT_CONVERTER_MANIFEST_CONFIG = {
|
|
|
41
41
|
hashAlgorithm: 'sha256',
|
|
42
42
|
optimizations: null,
|
|
43
43
|
conversion: null,
|
|
44
|
+
visionConfig: null,
|
|
45
|
+
audioConfig: null,
|
|
44
46
|
};
|
|
45
47
|
|
|
46
48
|
export const DEFAULT_CONVERTER_INFERENCE_CONFIG = {
|
|
47
|
-
|
|
49
|
+
session: null,
|
|
48
50
|
execution: null,
|
|
49
51
|
};
|
|
50
52
|
|
|
@@ -124,14 +126,14 @@ export function createConverterConfig(overrides) {
|
|
|
124
126
|
? { ...DEFAULT_CONVERTER_OUTPUT_CONFIG, ...overrides.output }
|
|
125
127
|
: { ...DEFAULT_CONVERTER_OUTPUT_CONFIG },
|
|
126
128
|
};
|
|
127
|
-
// V1 conversion configs place execution,
|
|
129
|
+
// V1 conversion configs place execution, session, and modelType at the top level.
|
|
128
130
|
// Pass them through so isV1Config() can detect the v1 format and resolveConversionPlanV1
|
|
129
131
|
// can read modelType.
|
|
130
|
-
if (
|
|
132
|
+
if (Object.prototype.hasOwnProperty.call(overrides, 'execution')) {
|
|
131
133
|
config.execution = overrides.execution;
|
|
132
134
|
}
|
|
133
|
-
if (
|
|
134
|
-
config.
|
|
135
|
+
if (Object.prototype.hasOwnProperty.call(overrides, 'session')) {
|
|
136
|
+
config.session = overrides.session;
|
|
135
137
|
}
|
|
136
138
|
if (typeof overrides.modelType === 'string') {
|
|
137
139
|
config.modelType = overrides.modelType;
|
|
@@ -15,10 +15,34 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import type { LoadingConfigSchema } from './loading.schema.js';
|
|
18
|
-
import type {
|
|
18
|
+
import type { ExecutionV1PatchSchema, ExecutionV1SessionSchema } from './execution-v1.schema.js';
|
|
19
19
|
import type { SharedRuntimeConfigSchema } from './shared-runtime.schema.js';
|
|
20
20
|
import type { EmulationConfigSchema } from './emulation.schema.js';
|
|
21
21
|
|
|
22
|
+
export interface RuntimeInferenceConfigSchema {
|
|
23
|
+
prompt?: string | null;
|
|
24
|
+
debugTokens?: boolean;
|
|
25
|
+
batching: Record<string, unknown>;
|
|
26
|
+
sampling: Record<string, unknown>;
|
|
27
|
+
compute: Record<string, unknown>;
|
|
28
|
+
tokenizer: Record<string, unknown>;
|
|
29
|
+
largeWeights: Record<string, unknown>;
|
|
30
|
+
kvcache: Record<string, unknown>;
|
|
31
|
+
diffusion: Record<string, unknown>;
|
|
32
|
+
energy: Record<string, unknown>;
|
|
33
|
+
moe: Record<string, unknown>;
|
|
34
|
+
speculative: Record<string, unknown>;
|
|
35
|
+
generation: Record<string, unknown>;
|
|
36
|
+
chatTemplate: Record<string, unknown>;
|
|
37
|
+
session: Partial<ExecutionV1SessionSchema>;
|
|
38
|
+
executionPatch: ExecutionV1PatchSchema | Record<string, unknown>;
|
|
39
|
+
kernelPath?: unknown;
|
|
40
|
+
kernelPathSource?: 'runtime' | 'config' | 'model' | 'manifest' | 'none';
|
|
41
|
+
kernelPathPolicy?: Record<string, unknown> | null;
|
|
42
|
+
modelOverrides?: Record<string, unknown>;
|
|
43
|
+
pipeline?: Record<string, unknown> | null;
|
|
44
|
+
}
|
|
45
|
+
|
|
22
46
|
/**
|
|
23
47
|
* Runtime configuration schema.
|
|
24
48
|
*
|
|
@@ -33,7 +57,7 @@ export interface RuntimeConfigSchema {
|
|
|
33
57
|
loading: LoadingConfigSchema;
|
|
34
58
|
|
|
35
59
|
/** Batching, sampling, tokenizer defaults */
|
|
36
|
-
inference:
|
|
60
|
+
inference: RuntimeInferenceConfigSchema;
|
|
37
61
|
|
|
38
62
|
/** NVIDIA superchip emulation settings */
|
|
39
63
|
emulation: EmulationConfigSchema;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { DEFAULT_LOADING_CONFIG } from './loading.schema.js';
|
|
2
|
-
import { DEFAULT_INFERENCE_DEFAULTS_CONFIG } from './inference-defaults.schema.js';
|
|
3
2
|
import { DEFAULT_SHARED_RUNTIME_CONFIG } from './shared-runtime.schema.js';
|
|
4
3
|
import { DEFAULT_EMULATION_CONFIG, createEmulationConfig } from './emulation.schema.js';
|
|
4
|
+
import { DEFAULT_KVCACHE_CONFIG } from './kvcache.schema.js';
|
|
5
|
+
import { DEFAULT_MOE_RUNTIME_CONFIG } from './moe.schema.js';
|
|
6
|
+
import { DEFAULT_SPECULATIVE_CONFIG } from './speculative.schema.js';
|
|
5
7
|
import { mergeEcosystemConfig } from './ecosystem.schema.js';
|
|
6
8
|
import {
|
|
7
9
|
chooseNullish,
|
|
@@ -16,10 +18,42 @@ import {
|
|
|
16
18
|
// Runtime Config (all non-model-specific settings)
|
|
17
19
|
// =============================================================================
|
|
18
20
|
|
|
21
|
+
export const DEFAULT_CHAT_TEMPLATE_CONFIG = Object.freeze({
|
|
22
|
+
enabled: null,
|
|
23
|
+
type: null,
|
|
24
|
+
thinking: false,
|
|
25
|
+
});
|
|
26
|
+
|
|
19
27
|
export const DEFAULT_RUNTIME_CONFIG = {
|
|
20
28
|
shared: DEFAULT_SHARED_RUNTIME_CONFIG,
|
|
21
29
|
loading: DEFAULT_LOADING_CONFIG,
|
|
22
|
-
inference:
|
|
30
|
+
inference: {
|
|
31
|
+
batching: {},
|
|
32
|
+
sampling: {
|
|
33
|
+
temperature: 1.0,
|
|
34
|
+
topP: 0.95,
|
|
35
|
+
topK: 50,
|
|
36
|
+
repetitionPenalty: 1.1,
|
|
37
|
+
greedyThreshold: 0.01,
|
|
38
|
+
repetitionPenaltyWindow: 100,
|
|
39
|
+
},
|
|
40
|
+
compute: {},
|
|
41
|
+
tokenizer: {},
|
|
42
|
+
largeWeights: {},
|
|
43
|
+
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
44
|
+
diffusion: {},
|
|
45
|
+
energy: {},
|
|
46
|
+
moe: DEFAULT_MOE_RUNTIME_CONFIG,
|
|
47
|
+
speculative: DEFAULT_SPECULATIVE_CONFIG,
|
|
48
|
+
generation: {
|
|
49
|
+
disableMultiTokenDecode: false,
|
|
50
|
+
},
|
|
51
|
+
chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
|
|
52
|
+
session: {
|
|
53
|
+
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
54
|
+
},
|
|
55
|
+
executionPatch: {},
|
|
56
|
+
},
|
|
23
57
|
emulation: DEFAULT_EMULATION_CONFIG,
|
|
24
58
|
};
|
|
25
59
|
|
|
@@ -167,7 +201,16 @@ function mergeLoadingConfig(
|
|
|
167
201
|
: { ...base.storage },
|
|
168
202
|
distribution: { ...base.distribution, ...overrides.distribution },
|
|
169
203
|
shardCache: { ...base.shardCache, ...overrides.shardCache },
|
|
170
|
-
memoryManagement:
|
|
204
|
+
memoryManagement: overrides.memoryManagement
|
|
205
|
+
? {
|
|
206
|
+
...base.memoryManagement,
|
|
207
|
+
...overrides.memoryManagement,
|
|
208
|
+
budget: {
|
|
209
|
+
...base.memoryManagement.budget,
|
|
210
|
+
...overrides.memoryManagement.budget,
|
|
211
|
+
},
|
|
212
|
+
}
|
|
213
|
+
: { ...base.memoryManagement },
|
|
171
214
|
prefetch: { ...base.prefetch, ...overrides.prefetch },
|
|
172
215
|
opfsPath: { ...base.opfsPath, ...overrides.opfsPath },
|
|
173
216
|
expertCache: { ...base.expertCache, ...overrides.expertCache },
|
|
@@ -189,6 +232,11 @@ function mergeInferenceConfig(
|
|
|
189
232
|
const overrideExecutionPatch = overrides.executionPatch ?? {};
|
|
190
233
|
const baseKernelPathPolicy = base.kernelPathPolicy ?? {};
|
|
191
234
|
const overrideKernelPathPolicy = overrides.kernelPathPolicy ?? {};
|
|
235
|
+
const baseDiffusion = base.diffusion ?? {};
|
|
236
|
+
const baseDiffusionDecode = baseDiffusion.decode ?? {};
|
|
237
|
+
const baseEnergy = base.energy ?? {};
|
|
238
|
+
const baseEnergyQuintel = baseEnergy.quintel ?? {};
|
|
239
|
+
const baseMoe = base.moe ?? {};
|
|
192
240
|
const hasRuntimeKernelProfiles = Object.prototype.hasOwnProperty.call(
|
|
193
241
|
overrideSessionCompute,
|
|
194
242
|
'kernelProfiles'
|
|
@@ -205,47 +253,47 @@ function mergeInferenceConfig(
|
|
|
205
253
|
kvcache: { ...base.kvcache, ...overrides.kvcache },
|
|
206
254
|
diffusion: overrides.diffusion
|
|
207
255
|
? {
|
|
208
|
-
...
|
|
256
|
+
...baseDiffusion,
|
|
209
257
|
...overrides.diffusion,
|
|
210
|
-
scheduler: { ...
|
|
211
|
-
latent: { ...
|
|
212
|
-
textEncoder: { ...
|
|
258
|
+
scheduler: { ...baseDiffusion.scheduler, ...overrides.diffusion.scheduler },
|
|
259
|
+
latent: { ...baseDiffusion.latent, ...overrides.diffusion.latent },
|
|
260
|
+
textEncoder: { ...baseDiffusion.textEncoder, ...overrides.diffusion.textEncoder },
|
|
213
261
|
decode: {
|
|
214
|
-
...
|
|
262
|
+
...baseDiffusionDecode,
|
|
215
263
|
...overrides.diffusion.decode,
|
|
216
|
-
tiling: { ...
|
|
264
|
+
tiling: { ...baseDiffusionDecode.tiling, ...overrides.diffusion.decode?.tiling },
|
|
217
265
|
},
|
|
218
|
-
swapper: { ...
|
|
219
|
-
quantization: { ...
|
|
266
|
+
swapper: { ...baseDiffusion.swapper, ...overrides.diffusion.swapper },
|
|
267
|
+
quantization: { ...baseDiffusion.quantization, ...overrides.diffusion.quantization },
|
|
220
268
|
}
|
|
221
|
-
: { ...
|
|
269
|
+
: { ...baseDiffusion },
|
|
222
270
|
energy: overrides.energy
|
|
223
271
|
? {
|
|
224
|
-
...
|
|
272
|
+
...baseEnergy,
|
|
225
273
|
...overrides.energy,
|
|
226
|
-
problem: overrides.energy.problem ??
|
|
227
|
-
state: { ...
|
|
228
|
-
init: { ...
|
|
229
|
-
target: { ...
|
|
230
|
-
loop: { ...
|
|
231
|
-
diagnostics: { ...
|
|
274
|
+
problem: overrides.energy.problem ?? baseEnergy.problem,
|
|
275
|
+
state: { ...baseEnergy.state, ...overrides.energy.state },
|
|
276
|
+
init: { ...baseEnergy.init, ...overrides.energy.init },
|
|
277
|
+
target: { ...baseEnergy.target, ...overrides.energy.target },
|
|
278
|
+
loop: { ...baseEnergy.loop, ...overrides.energy.loop },
|
|
279
|
+
diagnostics: { ...baseEnergy.diagnostics, ...overrides.energy.diagnostics },
|
|
232
280
|
quintel: overrides.energy.quintel
|
|
233
281
|
? {
|
|
234
|
-
...
|
|
282
|
+
...baseEnergyQuintel,
|
|
235
283
|
...overrides.energy.quintel,
|
|
236
|
-
rules: { ...
|
|
237
|
-
weights: { ...
|
|
238
|
-
clamp: { ...
|
|
284
|
+
rules: { ...baseEnergyQuintel.rules, ...overrides.energy.quintel.rules },
|
|
285
|
+
weights: { ...baseEnergyQuintel.weights, ...overrides.energy.quintel.weights },
|
|
286
|
+
clamp: { ...baseEnergyQuintel.clamp, ...overrides.energy.quintel.clamp },
|
|
239
287
|
}
|
|
240
|
-
: { ...
|
|
288
|
+
: { ...baseEnergyQuintel },
|
|
241
289
|
}
|
|
242
|
-
: { ...
|
|
290
|
+
: { ...baseEnergy },
|
|
243
291
|
moe: overrides.moe
|
|
244
292
|
? {
|
|
245
|
-
routing: { ...
|
|
246
|
-
cache: { ...
|
|
293
|
+
routing: { ...baseMoe.routing, ...overrides.moe.routing },
|
|
294
|
+
cache: { ...baseMoe.cache, ...overrides.moe.cache },
|
|
247
295
|
}
|
|
248
|
-
: { ...
|
|
296
|
+
: { ...baseMoe },
|
|
249
297
|
speculative: { ...base.speculative, ...overrides.speculative },
|
|
250
298
|
generation: { ...base.generation, ...overrides.generation },
|
|
251
299
|
pipeline: overrides.pipeline ?? base.pipeline,
|
|
@@ -269,6 +317,7 @@ function mergeInferenceConfig(
|
|
|
269
317
|
},
|
|
270
318
|
kvcache: replaceSubtree(overrideSession.kvcache, baseSession.kvcache),
|
|
271
319
|
decodeLoop: replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop),
|
|
320
|
+
perLayerInputs: replaceSubtree(overrideSession.perLayerInputs, baseSession.perLayerInputs),
|
|
272
321
|
},
|
|
273
322
|
executionPatch: mergeExecutionPatchLists(baseExecutionPatch, overrideExecutionPatch),
|
|
274
323
|
// Model-specific inference overrides (merged with manifest.inference at load time)
|
|
@@ -20,6 +20,13 @@ import type { KVCacheConfigSchema } from './kvcache.schema.js';
|
|
|
20
20
|
|
|
21
21
|
export type ExecutionV1Dtype = 'f16' | 'f32';
|
|
22
22
|
|
|
23
|
+
export interface ExecutionV1PrecisionSchema {
|
|
24
|
+
activationDtype?: ExecutionV1Dtype;
|
|
25
|
+
kvDtype?: ExecutionV1Dtype;
|
|
26
|
+
inputDtype?: ExecutionV1Dtype;
|
|
27
|
+
outputDtype?: ExecutionV1Dtype;
|
|
28
|
+
}
|
|
29
|
+
|
|
23
30
|
// === Kernel Declarations ===
|
|
24
31
|
|
|
25
32
|
/** A kernel declaration — defines a shader + entry + pinned digest. */
|
|
@@ -32,6 +39,8 @@ export interface ExecutionV1KernelSchema {
|
|
|
32
39
|
digest: string;
|
|
33
40
|
/** Pipeline override constants (optional, baked at pipeline creation) */
|
|
34
41
|
constants?: Record<string, number | boolean>;
|
|
42
|
+
/** Explicit per-step precision contract carried into inline kernel paths. */
|
|
43
|
+
precision?: ExecutionV1PrecisionSchema;
|
|
35
44
|
}
|
|
36
45
|
|
|
37
46
|
/** Map of kernel key → kernel declaration. */
|
|
@@ -72,7 +81,7 @@ export type ExecutionV1StepEntry = ExecutionV1StepTuple | ExecutionV1LayerGroupS
|
|
|
72
81
|
*/
|
|
73
82
|
export type ExecutionV1BoundaryStep = ExecutionV1StepTuple;
|
|
74
83
|
|
|
75
|
-
// === Session
|
|
84
|
+
// === Session ===
|
|
76
85
|
|
|
77
86
|
export interface ExecutionV1ComputeDefaultsSchema {
|
|
78
87
|
activationDtype: ExecutionV1Dtype;
|
|
@@ -81,16 +90,73 @@ export interface ExecutionV1ComputeDefaultsSchema {
|
|
|
81
90
|
outputDtype: ExecutionV1Dtype;
|
|
82
91
|
}
|
|
83
92
|
|
|
93
|
+
export type ReadbackMode = 'sequential' | 'overlapped' | 'auto';
|
|
94
|
+
|
|
84
95
|
export interface ExecutionV1DecodeLoopSchema {
|
|
85
96
|
batchSize: number;
|
|
86
97
|
stopCheckMode: 'per-token' | 'batch';
|
|
87
98
|
readbackInterval: number | null;
|
|
99
|
+
readbackMode: ReadbackMode;
|
|
100
|
+
/**
|
|
101
|
+
* Submit latency threshold in ms. Only consulted when readbackMode === "auto".
|
|
102
|
+
* When the GPU submit probe roundtrip exceeds this value, auto resolves to
|
|
103
|
+
* "overlapped"; otherwise resolves to "sequential".
|
|
104
|
+
*
|
|
105
|
+
* Set to `null` to disable auto (treat as sequential).
|
|
106
|
+
*/
|
|
107
|
+
submitLatencyThresholdMs: number | null;
|
|
88
108
|
ringTokens: number | null;
|
|
89
109
|
ringStop: number | null;
|
|
90
110
|
ringStaging: number | null;
|
|
91
111
|
disableCommandBatching?: boolean;
|
|
92
112
|
}
|
|
93
113
|
|
|
114
|
+
export type PerLayerInputMaterializationMode =
|
|
115
|
+
'auto'
|
|
116
|
+
| 'range_backed'
|
|
117
|
+
| 'cpu_resident'
|
|
118
|
+
| 'gpu_resident'
|
|
119
|
+
| 'gpu_split_tables';
|
|
120
|
+
|
|
121
|
+
export type PerLayerInputRowCacheMode = 'off' | 'lru';
|
|
122
|
+
|
|
123
|
+
export type PerLayerInputPrefetchMode = 'off' | 'next_token';
|
|
124
|
+
|
|
125
|
+
export type PerLayerInputGpuUploadMode = 'per_step_slices' | 'per_batch_slices';
|
|
126
|
+
export type PerLayerInputHotCacheMode = 'off' | 'prepared_tokens' | 'tokenizer_scores';
|
|
127
|
+
|
|
128
|
+
export interface ExecutionV1PerLayerInputsRowCacheSchema {
|
|
129
|
+
mode: PerLayerInputRowCacheMode;
|
|
130
|
+
maxRows: number;
|
|
131
|
+
maxBytes: number;
|
|
132
|
+
decodedDtype: ExecutionV1Dtype;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface ExecutionV1PerLayerInputsPrefetchSchema {
|
|
136
|
+
mode: PerLayerInputPrefetchMode;
|
|
137
|
+
rowsAhead: number;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export interface ExecutionV1PerLayerInputsGpuUploadSchema {
|
|
141
|
+
mode: PerLayerInputGpuUploadMode;
|
|
142
|
+
stagingRows: number;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export interface ExecutionV1PerLayerInputsHotCacheSchema {
|
|
146
|
+
mode: PerLayerInputHotCacheMode;
|
|
147
|
+
maxTokens: number;
|
|
148
|
+
maxBytes: number;
|
|
149
|
+
outputDtype: ExecutionV1Dtype;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export interface ExecutionV1PerLayerInputsSessionSchema {
|
|
153
|
+
materialization: PerLayerInputMaterializationMode;
|
|
154
|
+
rowCache: ExecutionV1PerLayerInputsRowCacheSchema;
|
|
155
|
+
prefetch: ExecutionV1PerLayerInputsPrefetchSchema;
|
|
156
|
+
gpuUpload: ExecutionV1PerLayerInputsGpuUploadSchema;
|
|
157
|
+
hotCache: ExecutionV1PerLayerInputsHotCacheSchema;
|
|
158
|
+
}
|
|
159
|
+
|
|
94
160
|
export interface ExecutionV1SelfSpeculationSchema {
|
|
95
161
|
mode: 'none' | 'self' | 'draft' | 'medusa';
|
|
96
162
|
tokens: number;
|
|
@@ -99,12 +165,13 @@ export interface ExecutionV1SelfSpeculationSchema {
|
|
|
99
165
|
rollbackOnReject: boolean;
|
|
100
166
|
}
|
|
101
167
|
|
|
102
|
-
export interface
|
|
168
|
+
export interface ExecutionV1SessionSchema {
|
|
103
169
|
compute: {
|
|
104
170
|
defaults: ExecutionV1ComputeDefaultsSchema;
|
|
105
171
|
};
|
|
106
172
|
kvcache: Partial<KVCacheConfigSchema> | null;
|
|
107
173
|
decodeLoop: ExecutionV1DecodeLoopSchema | null;
|
|
174
|
+
perLayerInputs: ExecutionV1PerLayerInputsSessionSchema;
|
|
108
175
|
speculation: ExecutionV1SelfSpeculationSchema | null;
|
|
109
176
|
}
|
|
110
177
|
|
|
@@ -153,8 +220,8 @@ export interface ExecutionV1ConfigSchema {
|
|
|
153
220
|
/** Schema discriminator — always "doppler.execution/v1" */
|
|
154
221
|
schema: 'doppler.execution/v1';
|
|
155
222
|
|
|
156
|
-
/** Session
|
|
157
|
-
|
|
223
|
+
/** Session policy (dtypes, KV cache, decode loop) */
|
|
224
|
+
session: ExecutionV1SessionSchema;
|
|
158
225
|
|
|
159
226
|
/** The execution graph */
|
|
160
227
|
execution: ExecutionV1GraphSchema;
|
|
@@ -227,8 +294,14 @@ export interface ExecutionV1ExpandedStepSchema {
|
|
|
227
294
|
// === Constants ===
|
|
228
295
|
|
|
229
296
|
export declare const EXECUTION_V1_SCHEMA_ID: string;
|
|
297
|
+
export declare const READBACK_MODES: readonly ['sequential', 'overlapped', 'auto'];
|
|
298
|
+
export declare const PER_LAYER_INPUT_MATERIALIZATION_MODES: readonly ['auto', 'range_backed', 'cpu_resident', 'gpu_resident'];
|
|
299
|
+
export declare const PER_LAYER_INPUT_ROW_CACHE_MODES: readonly ['off', 'lru'];
|
|
300
|
+
export declare const PER_LAYER_INPUT_PREFETCH_MODES: readonly ['off', 'next_token'];
|
|
301
|
+
export declare const PER_LAYER_INPUT_GPU_UPLOAD_MODES: readonly ['per_step_slices', 'per_batch_slices'];
|
|
230
302
|
export declare const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS: ExecutionV1ComputeDefaultsSchema;
|
|
231
|
-
export declare const
|
|
303
|
+
export declare const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION: ExecutionV1PerLayerInputsSessionSchema;
|
|
304
|
+
export declare const DEFAULT_EXECUTION_V1_SESSION: ExecutionV1SessionSchema;
|
|
232
305
|
export declare const DEFAULT_EXECUTION_V1_POLICIES: ExecutionV1PoliciesSchema;
|
|
233
306
|
export declare const DEFAULT_EXECUTION_V1_PATCH: ExecutionV1PatchSchema;
|
|
234
307
|
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
export const EXECUTION_V1_SCHEMA_ID = 'doppler.execution/v1';
|
|
6
6
|
|
|
7
7
|
const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/;
|
|
8
|
+
const EXECUTION_V1_DTYPE_SET = new Set(['f16', 'f32']);
|
|
8
9
|
|
|
9
10
|
export const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS = {
|
|
10
11
|
activationDtype: 'f16',
|
|
@@ -13,12 +14,57 @@ export const DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS = {
|
|
|
13
14
|
outputDtype: 'f16',
|
|
14
15
|
};
|
|
15
16
|
|
|
16
|
-
export const
|
|
17
|
+
export const READBACK_MODES = Object.freeze(['sequential', 'overlapped', 'auto']);
|
|
18
|
+
export const PER_LAYER_INPUT_MATERIALIZATION_MODES = Object.freeze([
|
|
19
|
+
'auto',
|
|
20
|
+
'range_backed',
|
|
21
|
+
'cpu_resident',
|
|
22
|
+
'gpu_resident',
|
|
23
|
+
'gpu_split_tables',
|
|
24
|
+
]);
|
|
25
|
+
export const PER_LAYER_INPUT_ROW_CACHE_MODES = Object.freeze(['off', 'lru']);
|
|
26
|
+
export const PER_LAYER_INPUT_PREFETCH_MODES = Object.freeze(['off', 'next_token']);
|
|
27
|
+
export const PER_LAYER_INPUT_GPU_UPLOAD_MODES = Object.freeze([
|
|
28
|
+
'per_step_slices',
|
|
29
|
+
'per_batch_slices',
|
|
30
|
+
]);
|
|
31
|
+
export const PER_LAYER_INPUT_HOT_CACHE_MODES = Object.freeze([
|
|
32
|
+
'off',
|
|
33
|
+
'prepared_tokens',
|
|
34
|
+
'tokenizer_scores',
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
export const DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION = {
|
|
38
|
+
materialization: 'auto',
|
|
39
|
+
rowCache: {
|
|
40
|
+
mode: 'lru',
|
|
41
|
+
maxRows: 256,
|
|
42
|
+
maxBytes: 134217728,
|
|
43
|
+
decodedDtype: 'f32',
|
|
44
|
+
},
|
|
45
|
+
prefetch: {
|
|
46
|
+
mode: 'next_token',
|
|
47
|
+
rowsAhead: 1,
|
|
48
|
+
},
|
|
49
|
+
gpuUpload: {
|
|
50
|
+
mode: 'per_step_slices',
|
|
51
|
+
stagingRows: 2,
|
|
52
|
+
},
|
|
53
|
+
hotCache: {
|
|
54
|
+
mode: 'prepared_tokens',
|
|
55
|
+
maxTokens: 1024,
|
|
56
|
+
maxBytes: 134217728,
|
|
57
|
+
outputDtype: 'f32',
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
export const DEFAULT_EXECUTION_V1_SESSION = {
|
|
17
62
|
compute: {
|
|
18
63
|
defaults: { ...DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS },
|
|
19
64
|
},
|
|
20
65
|
kvcache: null,
|
|
21
66
|
decodeLoop: null,
|
|
67
|
+
perLayerInputs: { ...DEFAULT_EXECUTION_V1_PER_LAYER_INPUTS_SESSION },
|
|
22
68
|
speculation: null,
|
|
23
69
|
};
|
|
24
70
|
|
|
@@ -64,6 +110,21 @@ function validateKernelMap(kernels, options = {}) {
|
|
|
64
110
|
if (decl.constants != null && typeof decl.constants !== 'object') {
|
|
65
111
|
throw new Error(`execution.kernels["${key}"].constants must be an object or null.`);
|
|
66
112
|
}
|
|
113
|
+
if (decl.precision != null) {
|
|
114
|
+
if (typeof decl.precision !== 'object' || Array.isArray(decl.precision)) {
|
|
115
|
+
throw new Error(`execution.kernels["${key}"].precision must be an object or null.`);
|
|
116
|
+
}
|
|
117
|
+
for (const field of ['activationDtype', 'kvDtype', 'inputDtype', 'outputDtype']) {
|
|
118
|
+
const value = decl.precision[field];
|
|
119
|
+
if (value === undefined) continue;
|
|
120
|
+
const normalized = String(value).trim().toLowerCase();
|
|
121
|
+
if (!EXECUTION_V1_DTYPE_SET.has(normalized)) {
|
|
122
|
+
throw new Error(
|
|
123
|
+
`execution.kernels["${key}"].precision.${field} must be "f16" or "f32"; got "${value}".`
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
67
128
|
}
|
|
68
129
|
}
|
|
69
130
|
|
|
@@ -121,7 +121,7 @@ export {
|
|
|
121
121
|
type ExecutionV1BoundaryStep,
|
|
122
122
|
type ExecutionV1ComputeDefaultsSchema,
|
|
123
123
|
type ExecutionV1DecodeLoopSchema,
|
|
124
|
-
type
|
|
124
|
+
type ExecutionV1SessionSchema,
|
|
125
125
|
type ExecutionV1PoliciesSchema,
|
|
126
126
|
type ExecutionV1GraphSchema,
|
|
127
127
|
type ExecutionV1ConfigSchema,
|
|
@@ -133,7 +133,7 @@ export {
|
|
|
133
133
|
type ExecutionV1ExpandedStepSchema,
|
|
134
134
|
EXECUTION_V1_SCHEMA_ID,
|
|
135
135
|
DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS,
|
|
136
|
-
|
|
136
|
+
DEFAULT_EXECUTION_V1_SESSION,
|
|
137
137
|
DEFAULT_EXECUTION_V1_POLICIES,
|
|
138
138
|
DEFAULT_EXECUTION_V1_PATCH,
|
|
139
139
|
isExecutionV1Digest,
|
|
@@ -231,6 +231,23 @@ export {
|
|
|
231
231
|
validateBrowserSuiteMetrics,
|
|
232
232
|
} from './browser-suite-metrics.schema.js';
|
|
233
233
|
|
|
234
|
+
// =============================================================================
|
|
235
|
+
// Support-Tier Registry Schema
|
|
236
|
+
// =============================================================================
|
|
237
|
+
export {
|
|
238
|
+
type SupportTier,
|
|
239
|
+
type SupportScope,
|
|
240
|
+
type ClaimVisibility,
|
|
241
|
+
type SupportSubsystemEntrySchema,
|
|
242
|
+
type SupportTierRegistrySchema,
|
|
243
|
+
SUPPORT_TIER_REGISTRY_SCHEMA_VERSION,
|
|
244
|
+
SUPPORT_TIERS,
|
|
245
|
+
SUPPORT_SCOPES,
|
|
246
|
+
CLAIM_VISIBILITY_LEVELS,
|
|
247
|
+
DEFAULT_SUPPORT_TIER_REGISTRY,
|
|
248
|
+
validateSupportTierRegistry,
|
|
249
|
+
} from './support-tiers.schema.js';
|
|
250
|
+
|
|
234
251
|
// =============================================================================
|
|
235
252
|
// Conversion Report Schema
|
|
236
253
|
// =============================================================================
|
|
@@ -285,6 +302,7 @@ export {
|
|
|
285
302
|
export {
|
|
286
303
|
// Types
|
|
287
304
|
type ShardCacheConfigSchema,
|
|
305
|
+
type LoaderMemoryBudgetConfigSchema,
|
|
288
306
|
type MemoryManagementConfigSchema,
|
|
289
307
|
type PrefetchConfigSchema,
|
|
290
308
|
type OpfsPathConfigSchema,
|
|
@@ -294,6 +312,7 @@ export {
|
|
|
294
312
|
// Defaults
|
|
295
313
|
DEFAULT_Q4K_LAYOUT,
|
|
296
314
|
DEFAULT_SHARD_CACHE_CONFIG,
|
|
315
|
+
DEFAULT_LOADER_MEMORY_BUDGET_CONFIG,
|
|
297
316
|
DEFAULT_MEMORY_MANAGEMENT_CONFIG,
|
|
298
317
|
DEFAULT_PREFETCH_CONFIG,
|
|
299
318
|
DEFAULT_OPFS_PATH_CONFIG,
|
|
@@ -328,6 +347,7 @@ export {
|
|
|
328
347
|
// =============================================================================
|
|
329
348
|
export {
|
|
330
349
|
// Types
|
|
350
|
+
type PlatformDetectionPattern,
|
|
331
351
|
type PlatformDetectionSchema,
|
|
332
352
|
type MemoryHintsSchema,
|
|
333
353
|
type PlatformSchema,
|
|
@@ -353,33 +373,6 @@ export {
|
|
|
353
373
|
DEFAULT_STORAGE_FULL_CONFIG,
|
|
354
374
|
} from './storage.schema.js';
|
|
355
375
|
|
|
356
|
-
// =============================================================================
|
|
357
|
-
// Inference Defaults Schema
|
|
358
|
-
// =============================================================================
|
|
359
|
-
export {
|
|
360
|
-
// Types
|
|
361
|
-
type BatchingDefaultsSchema,
|
|
362
|
-
type ComputeDefaultsSchema,
|
|
363
|
-
type LargeWeightConfigSchema,
|
|
364
|
-
type SamplingDefaultsSchema,
|
|
365
|
-
type GenerationDefaultsSchema,
|
|
366
|
-
type TokenizerDefaultsSchema,
|
|
367
|
-
type KernelPathPolicySchema,
|
|
368
|
-
type InferenceDefaultsConfigSchema,
|
|
369
|
-
type ModelInferenceOverrides,
|
|
370
|
-
|
|
371
|
-
// Defaults
|
|
372
|
-
DEFAULT_BATCHING_DEFAULTS,
|
|
373
|
-
DEFAULT_COMPUTE_DEFAULTS,
|
|
374
|
-
DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
375
|
-
DEFAULT_SAMPLING_DEFAULTS,
|
|
376
|
-
DEFAULT_GENERATION_CONFIG,
|
|
377
|
-
DEFAULT_TOKENIZER_DEFAULTS,
|
|
378
|
-
DEFAULT_KERNEL_PATH_POLICY,
|
|
379
|
-
DEFAULT_INFERENCE_DEFAULTS_CONFIG,
|
|
380
|
-
DEFAULT_MODEL_INFERENCE_DEFAULTS,
|
|
381
|
-
} from './inference-defaults.schema.js';
|
|
382
|
-
|
|
383
376
|
// =============================================================================
|
|
384
377
|
// Distribution Schema
|
|
385
378
|
// =============================================================================
|