doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
import { trace } from '../../debug/index.js';
|
|
4
|
+
import {
|
|
5
|
+
getSourceTransformSpec,
|
|
6
|
+
hasSourceTransform,
|
|
7
|
+
materializeTensorSourceTransform,
|
|
8
|
+
} from './source-transform.js';
|
|
4
9
|
|
|
5
10
|
function resolveSpanShardIndex(span, name, spanIndex) {
|
|
6
11
|
const shardIndex = typeof span?.shardIndex === 'number'
|
|
@@ -48,31 +53,179 @@ function validateLocationField(location, field, name) {
|
|
|
48
53
|
return value;
|
|
49
54
|
}
|
|
50
55
|
|
|
56
|
+
function getPhysicalChunks(location, name) {
|
|
57
|
+
const spans = getLocationSpans(location);
|
|
58
|
+
if (spans) {
|
|
59
|
+
return spans.map((span, spanIndex) => ({
|
|
60
|
+
shardIndex: resolveSpanShardIndex(span, name, spanIndex),
|
|
61
|
+
offset: validateSpanField(span.offset, 'offset', name, spanIndex),
|
|
62
|
+
size: validateSpanField(span.size, 'size', name, spanIndex),
|
|
63
|
+
}));
|
|
64
|
+
}
|
|
65
|
+
return [{
|
|
66
|
+
shardIndex: resolveLocationShardIndex(location, name),
|
|
67
|
+
offset: validateLocationField(location, 'offset', name),
|
|
68
|
+
size: validateLocationField(location, 'size', name),
|
|
69
|
+
}];
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function assembleLocationBytes(location, name, label, loadShard, loadShardRange = null) {
|
|
73
|
+
const chunks = getPhysicalChunks(location, name);
|
|
74
|
+
const parts = await Promise.all(chunks.map(async (chunk) => {
|
|
75
|
+
if (loadShardRange) {
|
|
76
|
+
const data = await loadShardRange(chunk.shardIndex, chunk.offset, chunk.size);
|
|
77
|
+
if (chunk.size > data.byteLength) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
`[DopplerLoader] Shard ${chunk.shardIndex} too small for tensor "${name}" ${label}.`
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
return new Uint8Array(data, 0, chunk.size);
|
|
83
|
+
}
|
|
84
|
+
const data = await loadShard(chunk.shardIndex);
|
|
85
|
+
if (chunk.offset + chunk.size > data.byteLength) {
|
|
86
|
+
throw new Error(
|
|
87
|
+
`[DopplerLoader] Shard ${chunk.shardIndex} too small for tensor "${name}" ${label}.`
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return new Uint8Array(data, chunk.offset, chunk.size);
|
|
91
|
+
}));
|
|
92
|
+
const totalSize = parts.reduce((sum, part) => sum + part.byteLength, 0);
|
|
93
|
+
const combined = new Uint8Array(totalSize);
|
|
94
|
+
let offset = 0;
|
|
95
|
+
for (const part of parts) {
|
|
96
|
+
combined.set(part, offset);
|
|
97
|
+
offset += part.byteLength;
|
|
98
|
+
}
|
|
99
|
+
return combined;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function loadLocationRange(location, name, label, byteOffset, byteLength, loadShardRange) {
|
|
103
|
+
const chunks = getPhysicalChunks(location, name);
|
|
104
|
+
const totalSize = chunks.reduce((sum, chunk) => sum + chunk.size, 0);
|
|
105
|
+
if (byteOffset + byteLength > totalSize) {
|
|
106
|
+
throw new Error(
|
|
107
|
+
`[DopplerLoader] Tensor "${name}" ${label} range (${byteOffset}..${byteOffset + byteLength}) exceeds size ${totalSize}.`
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const combined = new Uint8Array(byteLength);
|
|
112
|
+
let logicalOffset = 0;
|
|
113
|
+
let writeOffset = 0;
|
|
114
|
+
const rangeEnd = byteOffset + byteLength;
|
|
115
|
+
|
|
116
|
+
for (const chunk of chunks) {
|
|
117
|
+
const chunkStart = logicalOffset;
|
|
118
|
+
const chunkEnd = chunkStart + chunk.size;
|
|
119
|
+
logicalOffset = chunkEnd;
|
|
120
|
+
|
|
121
|
+
if (rangeEnd <= chunkStart || byteOffset >= chunkEnd) {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const start = Math.max(byteOffset, chunkStart);
|
|
126
|
+
const end = Math.min(rangeEnd, chunkEnd);
|
|
127
|
+
const localOffset = chunk.offset + (start - chunkStart);
|
|
128
|
+
const localSize = end - start;
|
|
129
|
+
const data = await loadShardRange(chunk.shardIndex, localOffset, localSize);
|
|
130
|
+
if (localSize > data.byteLength) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
`[DopplerLoader] Shard ${chunk.shardIndex} too small for tensor "${name}" ${label} range.`
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
combined.set(new Uint8Array(data, 0, localSize), writeOffset);
|
|
136
|
+
writeOffset += localSize;
|
|
137
|
+
|
|
138
|
+
if (writeOffset === byteLength) {
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (writeOffset !== byteLength) {
|
|
144
|
+
throw new Error(
|
|
145
|
+
`[DopplerLoader] Tensor "${name}" short ${label} range read: got ${writeOffset}, expected ${byteLength}.`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return combined;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function getSourceTransformScaleLocation(location, name) {
|
|
153
|
+
const scaleLocation = location?.sourceTransform?.scaleSource;
|
|
154
|
+
if (!scaleLocation || typeof scaleLocation !== 'object') {
|
|
155
|
+
throw new Error(
|
|
156
|
+
`[DopplerLoader] Tensor "${name}" sourceTransform is missing scaleSource.`
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
return scaleLocation;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function getSourceTransformSumLocation(location) {
|
|
163
|
+
const transform = location?.sourceTransform;
|
|
164
|
+
const sumLocation = transform?.kind === 'litert_axis_dequant'
|
|
165
|
+
? transform?.sumSource
|
|
166
|
+
: transform?.rowSumSource;
|
|
167
|
+
if (!sumLocation || typeof sumLocation !== 'object') {
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
return sumLocation;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function materializeLocationBytes(rawBytes, location, name, loadShard, loadShardRange = null, options = {}) {
|
|
174
|
+
if (!hasSourceTransform(location)) {
|
|
175
|
+
return rawBytes;
|
|
176
|
+
}
|
|
177
|
+
if (
|
|
178
|
+
location.sourceTransform.kind === 'litert_rowwise_dequant'
|
|
179
|
+
|| location.sourceTransform.kind === 'litert_axis_dequant'
|
|
180
|
+
) {
|
|
181
|
+
const scaleLocation = getSourceTransformScaleLocation(location, name);
|
|
182
|
+
const scaleBytes = options.scaleBytes instanceof Uint8Array
|
|
183
|
+
? options.scaleBytes
|
|
184
|
+
: await assembleLocationBytes(scaleLocation, name, 'scale companion', loadShard, loadShardRange);
|
|
185
|
+
const sumLocation = getSourceTransformSumLocation(location);
|
|
186
|
+
const sumBytes = options.sumBytes instanceof Uint8Array
|
|
187
|
+
? options.sumBytes
|
|
188
|
+
: options.rowSumBytes instanceof Uint8Array
|
|
189
|
+
? options.rowSumBytes
|
|
190
|
+
: (
|
|
191
|
+
sumLocation
|
|
192
|
+
? await assembleLocationBytes(sumLocation, name, 'sum companion', loadShard, loadShardRange)
|
|
193
|
+
: null
|
|
194
|
+
);
|
|
195
|
+
return materializeTensorSourceTransform(rawBytes, location, name, {
|
|
196
|
+
scaleBytes,
|
|
197
|
+
sumBytes,
|
|
198
|
+
rowSumBytes: sumBytes,
|
|
199
|
+
rowStart: options.rowStart ?? null,
|
|
200
|
+
rowCount: options.rowCount ?? null,
|
|
201
|
+
storageColumnStart: options.storageColumnStart ?? null,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
return materializeTensorSourceTransform(rawBytes, location, name);
|
|
205
|
+
}
|
|
206
|
+
|
|
51
207
|
export async function assembleShardData(location, name, loadShard, loadShardRange = null) {
|
|
52
208
|
const spans = getLocationSpans(location);
|
|
53
209
|
if (spans) {
|
|
54
210
|
trace.loader(`Assembling tensor "${name}" from ${spans.length} spans`);
|
|
55
211
|
|
|
56
|
-
const chunks = await Promise.all(
|
|
57
|
-
const shardIndex = resolveSpanShardIndex(span, name, spanIndex);
|
|
58
|
-
const offset = validateSpanField(span.offset, 'offset', name, spanIndex);
|
|
59
|
-
const size = validateSpanField(span.size, 'size', name, spanIndex);
|
|
212
|
+
const chunks = await Promise.all(getPhysicalChunks(location, name).map(async (chunk) => {
|
|
60
213
|
if (loadShardRange) {
|
|
61
|
-
const data = await loadShardRange(shardIndex, offset, size);
|
|
62
|
-
if (size > data.byteLength) {
|
|
214
|
+
const data = await loadShardRange(chunk.shardIndex, chunk.offset, chunk.size);
|
|
215
|
+
if (chunk.size > data.byteLength) {
|
|
63
216
|
throw new Error(
|
|
64
|
-
`[DopplerLoader] Shard ${shardIndex} too small for tensor "${name}" span.`
|
|
217
|
+
`[DopplerLoader] Shard ${chunk.shardIndex} too small for tensor "${name}" span.`
|
|
65
218
|
);
|
|
66
219
|
}
|
|
67
|
-
return new Uint8Array(data, 0, size);
|
|
220
|
+
return new Uint8Array(data, 0, chunk.size);
|
|
68
221
|
}
|
|
69
|
-
const data = await loadShard(shardIndex);
|
|
70
|
-
if (offset + size > data.byteLength) {
|
|
222
|
+
const data = await loadShard(chunk.shardIndex);
|
|
223
|
+
if (chunk.offset + chunk.size > data.byteLength) {
|
|
71
224
|
throw new Error(
|
|
72
|
-
`[DopplerLoader] Shard ${shardIndex} too small for tensor "${name}" span.`
|
|
225
|
+
`[DopplerLoader] Shard ${chunk.shardIndex} too small for tensor "${name}" span.`
|
|
73
226
|
);
|
|
74
227
|
}
|
|
75
|
-
return new Uint8Array(data, offset, size);
|
|
228
|
+
return new Uint8Array(data, chunk.offset, chunk.size);
|
|
76
229
|
}));
|
|
77
230
|
const totalSize = chunks.reduce((s, c) => s + c.length, 0);
|
|
78
231
|
if (Number.isInteger(location?.size) && totalSize !== location.size) {
|
|
@@ -86,7 +239,7 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
86
239
|
combined.set(chunk, offset);
|
|
87
240
|
offset += chunk.length;
|
|
88
241
|
}
|
|
89
|
-
return combined;
|
|
242
|
+
return materializeLocationBytes(combined, location, name, loadShard, loadShardRange);
|
|
90
243
|
}
|
|
91
244
|
|
|
92
245
|
// Single shard - use view to avoid copying
|
|
@@ -100,7 +253,8 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
100
253
|
`[DopplerLoader] Shard ${shardIndex} too small for tensor "${name}" (offset=${offset}, size=${size}, shard=${slice.byteLength})`
|
|
101
254
|
);
|
|
102
255
|
}
|
|
103
|
-
|
|
256
|
+
const bytes = new Uint8Array(slice, 0, size);
|
|
257
|
+
return materializeLocationBytes(bytes, location, name, loadShard, loadShardRange);
|
|
104
258
|
}
|
|
105
259
|
|
|
106
260
|
const fullShard = await loadShard(shardIndex);
|
|
@@ -109,5 +263,119 @@ export async function assembleShardData(location, name, loadShard, loadShardRang
|
|
|
109
263
|
`[DopplerLoader] Shard ${shardIndex} too small for tensor "${name}" (offset=${offset}, size=${size}, shard=${fullShard.byteLength})`
|
|
110
264
|
);
|
|
111
265
|
}
|
|
112
|
-
|
|
266
|
+
const bytes = new Uint8Array(fullShard, offset, size);
|
|
267
|
+
return materializeLocationBytes(bytes, location, name, loadShard, loadShardRange);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export async function loadTensorRange(location, name, byteOffset, byteLength, loadShardRange) {
|
|
271
|
+
if (typeof loadShardRange !== 'function') {
|
|
272
|
+
throw new Error(`[DopplerLoader] Tensor "${name}" range loading requires loadShardRange().`);
|
|
273
|
+
}
|
|
274
|
+
if (!Number.isInteger(byteOffset) || byteOffset < 0) {
|
|
275
|
+
throw new Error(`[DopplerLoader] Tensor "${name}" has invalid byteOffset ${byteOffset}.`);
|
|
276
|
+
}
|
|
277
|
+
if (!Number.isInteger(byteLength) || byteLength < 0) {
|
|
278
|
+
throw new Error(`[DopplerLoader] Tensor "${name}" has invalid byteLength ${byteLength}.`);
|
|
279
|
+
}
|
|
280
|
+
if (byteLength === 0) {
|
|
281
|
+
return new Uint8Array(0);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (hasSourceTransform(location)) {
|
|
285
|
+
const spec = getSourceTransformSpec(location, name);
|
|
286
|
+
if (
|
|
287
|
+
(spec?.kind === 'litert_rowwise_dequant' || spec?.kind === 'litert_axis_dequant')
|
|
288
|
+
&& (byteOffset % spec.targetRowBytes) === 0
|
|
289
|
+
&& (byteLength % spec.targetRowBytes) === 0
|
|
290
|
+
) {
|
|
291
|
+
const rowStart = byteOffset / spec.targetRowBytes;
|
|
292
|
+
const rowCount = byteLength / spec.targetRowBytes;
|
|
293
|
+
const scaleBytes = await loadLocationRange(
|
|
294
|
+
getSourceTransformScaleLocation(location, name),
|
|
295
|
+
name,
|
|
296
|
+
'LiteRT transformed scale',
|
|
297
|
+
rowStart * spec.scaleRowBytes,
|
|
298
|
+
rowCount * spec.scaleRowBytes,
|
|
299
|
+
loadShardRange
|
|
300
|
+
);
|
|
301
|
+
const sumLocation = getSourceTransformSumLocation(location);
|
|
302
|
+
const sumBytes = sumLocation
|
|
303
|
+
? await loadLocationRange(
|
|
304
|
+
sumLocation,
|
|
305
|
+
name,
|
|
306
|
+
'LiteRT transformed sum',
|
|
307
|
+
rowStart * 4,
|
|
308
|
+
rowCount * 4,
|
|
309
|
+
loadShardRange
|
|
310
|
+
)
|
|
311
|
+
: null;
|
|
312
|
+
if (spec.kind === 'litert_rowwise_dequant') {
|
|
313
|
+
const raw = await loadLocationRange(
|
|
314
|
+
location,
|
|
315
|
+
name,
|
|
316
|
+
'LiteRT transformed raw',
|
|
317
|
+
rowStart * spec.rawRowBytes,
|
|
318
|
+
rowCount * spec.rawRowBytes,
|
|
319
|
+
loadShardRange
|
|
320
|
+
);
|
|
321
|
+
return materializeTensorSourceTransform(raw, location, name, {
|
|
322
|
+
scaleBytes,
|
|
323
|
+
rowSumBytes: sumBytes,
|
|
324
|
+
rowStart,
|
|
325
|
+
rowCount,
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (spec.quantAxis === 1) {
|
|
330
|
+
const raw = await loadLocationRange(
|
|
331
|
+
location,
|
|
332
|
+
name,
|
|
333
|
+
'LiteRT transformed raw',
|
|
334
|
+
rowStart * spec.rawStorageRowBytes,
|
|
335
|
+
rowCount * spec.rawStorageRowBytes,
|
|
336
|
+
loadShardRange
|
|
337
|
+
);
|
|
338
|
+
return materializeTensorSourceTransform(raw, location, name, {
|
|
339
|
+
scaleBytes,
|
|
340
|
+
sumBytes,
|
|
341
|
+
rowStart,
|
|
342
|
+
rowCount,
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const storageColumnStart = Math.floor(rowStart / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
347
|
+
const storageColumnEnd = Math.ceil((rowStart + rowCount) / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
348
|
+
const rawSliceRowBytes = (storageColumnEnd - storageColumnStart) / spec.storageValuesPerByte;
|
|
349
|
+
const raw = new Uint8Array(spec.storageRows * rawSliceRowBytes);
|
|
350
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
351
|
+
const rowBytes = await loadLocationRange(
|
|
352
|
+
location,
|
|
353
|
+
name,
|
|
354
|
+
`LiteRT transformed raw storage row ${storageRow}`,
|
|
355
|
+
storageRow * spec.rawStorageRowBytes + (storageColumnStart / spec.storageValuesPerByte),
|
|
356
|
+
rawSliceRowBytes,
|
|
357
|
+
loadShardRange
|
|
358
|
+
);
|
|
359
|
+
raw.set(rowBytes, storageRow * rawSliceRowBytes);
|
|
360
|
+
}
|
|
361
|
+
return materializeTensorSourceTransform(raw, location, name, {
|
|
362
|
+
scaleBytes,
|
|
363
|
+
sumBytes,
|
|
364
|
+
rowStart,
|
|
365
|
+
rowCount,
|
|
366
|
+
storageColumnStart,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
const raw = await assembleLocationBytes(location, name, 'transformed tensor', null, loadShardRange);
|
|
370
|
+
const transformed = await materializeLocationBytes(raw, location, name, null, loadShardRange);
|
|
371
|
+
if (byteOffset + byteLength > transformed.byteLength) {
|
|
372
|
+
throw new Error(
|
|
373
|
+
`[DopplerLoader] Tensor "${name}" transformed range (${byteOffset}..${byteOffset + byteLength}) ` +
|
|
374
|
+
`exceeds size ${transformed.byteLength}.`
|
|
375
|
+
);
|
|
376
|
+
}
|
|
377
|
+
return transformed.slice(byteOffset, byteOffset + byteLength);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
return loadLocationRange(location, name, 'tensor', byteOffset, byteLength, loadShardRange);
|
|
113
381
|
}
|
|
@@ -188,6 +188,10 @@ export declare const acquireBuffer: (size: number, usage?: GPUBufferUsageFlags,
|
|
|
188
188
|
export declare const releaseBuffer: (buffer: GPUBuffer) => void;
|
|
189
189
|
export declare const discardBuffer: (buffer: GPUBuffer) => void;
|
|
190
190
|
export declare const isBufferActive: (buffer: GPUBuffer) => boolean;
|
|
191
|
+
export declare function markPersistentBuffer(buffer: GPUBuffer): void;
|
|
192
|
+
export declare function unmarkPersistentBuffer(buffer: GPUBuffer): void;
|
|
193
|
+
export declare function isPersistentBuffer(buffer: GPUBuffer): boolean;
|
|
194
|
+
export declare class PersistentBufferSet extends Set<GPUBuffer> {}
|
|
191
195
|
export declare const getBufferRequestedSize: (buffer: GPUBuffer) => number;
|
|
192
196
|
export declare const uploadData: (buffer: GPUBuffer, data: ArrayBuffer | ArrayBufferView, offset?: number) => void;
|
|
193
197
|
export declare const readBuffer: (buffer: GPUBuffer, size?: number) => Promise<ArrayBuffer>;
|
|
@@ -778,6 +778,7 @@ export class BufferPool {
|
|
|
778
778
|
|
|
779
779
|
let globalPool = null;
|
|
780
780
|
let globalPoolEpoch = -1;
|
|
781
|
+
const persistentBuffers = new WeakSet();
|
|
781
782
|
|
|
782
783
|
|
|
783
784
|
export function getBufferPool() {
|
|
@@ -836,6 +837,41 @@ export const discardBuffer = (buffer) => getBufferPool().discard(buffer);
|
|
|
836
837
|
export const isBufferActive = (buffer) =>
|
|
837
838
|
getBufferPool().isActiveBuffer(buffer);
|
|
838
839
|
|
|
840
|
+
export function markPersistentBuffer(buffer) {
|
|
841
|
+
if (buffer && typeof buffer === 'object') {
|
|
842
|
+
persistentBuffers.add(buffer);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
export function unmarkPersistentBuffer(buffer) {
|
|
847
|
+
if (buffer && typeof buffer === 'object') {
|
|
848
|
+
persistentBuffers.delete(buffer);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
export function isPersistentBuffer(buffer) {
|
|
853
|
+
return !!buffer && typeof buffer === 'object' && persistentBuffers.has(buffer);
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
export class PersistentBufferSet extends Set {
|
|
857
|
+
add(buffer) {
|
|
858
|
+
markPersistentBuffer(buffer);
|
|
859
|
+
return super.add(buffer);
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
delete(buffer) {
|
|
863
|
+
unmarkPersistentBuffer(buffer);
|
|
864
|
+
return super.delete(buffer);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
clear() {
|
|
868
|
+
for (const buffer of this) {
|
|
869
|
+
unmarkPersistentBuffer(buffer);
|
|
870
|
+
}
|
|
871
|
+
return super.clear();
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
839
875
|
export const getBufferRequestedSize = (buffer) =>
|
|
840
876
|
getBufferPool().getRequestedSize(buffer);
|
|
841
877
|
|
|
@@ -6,9 +6,13 @@ export interface InferenceExecutionRulesContractArtifact {
|
|
|
6
6
|
errors: string[];
|
|
7
7
|
stats: {
|
|
8
8
|
decodeRecorderRules: number;
|
|
9
|
+
profileDecodeRecorderRules: number;
|
|
9
10
|
batchDecodeRules: number;
|
|
11
|
+
maxBatchDecodeTokenRules: number;
|
|
10
12
|
decodeRecorderContexts: number;
|
|
13
|
+
profileDecodeRecorderContexts: number;
|
|
11
14
|
batchDecodeContexts: number;
|
|
15
|
+
maxBatchDecodeTokenContexts: number;
|
|
12
16
|
};
|
|
13
17
|
}
|
|
14
18
|
|
|
@@ -65,7 +65,15 @@ function batchDecodeSemantic(context) {
|
|
|
65
65
|
&& context.disableMultiTokenDecode !== true
|
|
66
66
|
&& context.disableCommandBatching !== true
|
|
67
67
|
&& context.isBdpaPagedLayout !== true
|
|
68
|
-
&& context.finitenessFallbackWindowOpen !== true
|
|
68
|
+
&& context.finitenessFallbackWindowOpen !== true
|
|
69
|
+
&& (
|
|
70
|
+
context.hasLinearAttentionLayers !== true
|
|
71
|
+
|| context.hasRangeBackedPerLayerInputs !== true
|
|
72
|
+
)
|
|
73
|
+
&& (
|
|
74
|
+
context.hasRangeBackedPerLayerInputs !== true
|
|
75
|
+
|| context.selfSpeculationEnabled !== true
|
|
76
|
+
);
|
|
69
77
|
}
|
|
70
78
|
|
|
71
79
|
function enumerateDecodeRecorderContexts() {
|
|
@@ -100,15 +108,24 @@ function enumerateBatchDecodeContexts() {
|
|
|
100
108
|
for (const disableCommandBatching of values) {
|
|
101
109
|
for (const isBdpaPagedLayout of values) {
|
|
102
110
|
for (const finitenessFallbackWindowOpen of values) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
111
|
+
for (const hasLinearAttentionLayers of values) {
|
|
112
|
+
for (const selfSpeculationEnabled of values) {
|
|
113
|
+
for (const hasRangeBackedPerLayerInputs of values) {
|
|
114
|
+
contexts.push({
|
|
115
|
+
batchSize,
|
|
116
|
+
useGPU,
|
|
117
|
+
gpuSamplingAvailable,
|
|
118
|
+
disableMultiTokenDecode,
|
|
119
|
+
disableCommandBatching,
|
|
120
|
+
isBdpaPagedLayout,
|
|
121
|
+
finitenessFallbackWindowOpen,
|
|
122
|
+
hasLinearAttentionLayers,
|
|
123
|
+
selfSpeculationEnabled,
|
|
124
|
+
hasRangeBackedPerLayerInputs,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
112
129
|
}
|
|
113
130
|
}
|
|
114
131
|
}
|
|
@@ -119,25 +136,58 @@ function enumerateBatchDecodeContexts() {
|
|
|
119
136
|
return contexts;
|
|
120
137
|
}
|
|
121
138
|
|
|
122
|
-
function
|
|
139
|
+
function maxBatchDecodeTokensSemantic(context) {
|
|
140
|
+
if (context.hasHotVocabularyBatchDecode === true) {
|
|
141
|
+
return 1;
|
|
142
|
+
}
|
|
143
|
+
if (context.hasLinearAttentionLayers === true) {
|
|
144
|
+
return 32;
|
|
145
|
+
}
|
|
146
|
+
return context.hasGpuSplitPerLayerInputs === true ? 4 : null;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function enumerateMaxBatchDecodeTokenContexts() {
|
|
150
|
+
return [
|
|
151
|
+
{ hasHotVocabularyBatchDecode: true, hasGpuSplitPerLayerInputs: false, hasLinearAttentionLayers: false },
|
|
152
|
+
{ hasHotVocabularyBatchDecode: false, hasGpuSplitPerLayerInputs: false, hasLinearAttentionLayers: true },
|
|
153
|
+
{ hasHotVocabularyBatchDecode: false, hasGpuSplitPerLayerInputs: true, hasLinearAttentionLayers: false },
|
|
154
|
+
{ hasHotVocabularyBatchDecode: false, hasGpuSplitPerLayerInputs: false, hasLinearAttentionLayers: false },
|
|
155
|
+
];
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function checkRuleShape(rules, expectedMatches, label) {
|
|
123
159
|
if (!Array.isArray(rules)) {
|
|
124
160
|
return {
|
|
125
161
|
ok: false,
|
|
126
162
|
errors: [`[ExecutionRulesContract] ${label} must be an array.`],
|
|
127
163
|
};
|
|
128
164
|
}
|
|
129
|
-
if (
|
|
165
|
+
if (!Array.isArray(expectedMatches) || expectedMatches.length === 0) {
|
|
130
166
|
return {
|
|
131
167
|
ok: false,
|
|
132
|
-
errors: [`[ExecutionRulesContract] ${label} must
|
|
168
|
+
errors: [`[ExecutionRulesContract] ${label} expectedMatches must be a non-empty array.`],
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
if (rules.length !== expectedMatches.length + 1) {
|
|
172
|
+
return {
|
|
173
|
+
ok: false,
|
|
174
|
+
errors: [
|
|
175
|
+
`[ExecutionRulesContract] ${label} must contain exactly ${expectedMatches.length + 1} rules; got ${rules.length}.`,
|
|
176
|
+
],
|
|
133
177
|
};
|
|
134
178
|
}
|
|
135
|
-
const [firstRule, secondRule] = rules;
|
|
136
179
|
const errors = [];
|
|
137
|
-
|
|
138
|
-
|
|
180
|
+
for (let index = 0; index < expectedMatches.length; index += 1) {
|
|
181
|
+
const rule = rules[index];
|
|
182
|
+
if (!matchesExactObject(rule?.match, expectedMatches[index]) || rule?.value !== true) {
|
|
183
|
+
errors.push(
|
|
184
|
+
`[ExecutionRulesContract] ${label} rule ${index + 1} drifted from the expected enabling predicate.`
|
|
185
|
+
);
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
139
188
|
}
|
|
140
|
-
|
|
189
|
+
const fallbackRule = rules[rules.length - 1];
|
|
190
|
+
if (!matchesExactObject(fallbackRule?.match, {}) || fallbackRule?.value !== false) {
|
|
141
191
|
errors.push(`[ExecutionRulesContract] ${label} fallback rule must be { match: {}, value: false }.`);
|
|
142
192
|
}
|
|
143
193
|
return {
|
|
@@ -172,15 +222,16 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
172
222
|
const decodeRules = ruleGroup?.decodeRecorderEnabled;
|
|
173
223
|
const profileDecodeRules = ruleGroup?.profileDecodeRecorderEnabled;
|
|
174
224
|
const batchRules = ruleGroup?.batchDecodeEnabled;
|
|
225
|
+
const maxBatchDecodeTokenRules = ruleGroup?.maxBatchDecodeTokens;
|
|
175
226
|
|
|
176
227
|
const decodeShape = checkRuleShape(
|
|
177
228
|
decodeRules,
|
|
178
|
-
{
|
|
229
|
+
[{
|
|
179
230
|
hasDevice: true,
|
|
180
231
|
debug: false,
|
|
181
232
|
disableCommandBatching: false,
|
|
182
233
|
kvLayout: { neq: 'bdpa_paged' },
|
|
183
|
-
},
|
|
234
|
+
}],
|
|
184
235
|
'decodeRecorderEnabled'
|
|
185
236
|
);
|
|
186
237
|
errors.push(...decodeShape.errors);
|
|
@@ -205,11 +256,11 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
205
256
|
|
|
206
257
|
const profileDecodeShape = checkRuleShape(
|
|
207
258
|
profileDecodeRules,
|
|
208
|
-
{
|
|
259
|
+
[{
|
|
209
260
|
hasDevice: true,
|
|
210
261
|
debug: false,
|
|
211
262
|
kvLayout: { neq: 'bdpa_paged' },
|
|
212
|
-
},
|
|
263
|
+
}],
|
|
213
264
|
'profileDecodeRecorderEnabled'
|
|
214
265
|
);
|
|
215
266
|
errors.push(...profileDecodeShape.errors);
|
|
@@ -238,15 +289,38 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
238
289
|
|
|
239
290
|
const batchShape = checkRuleShape(
|
|
240
291
|
batchRules,
|
|
241
|
-
{
|
|
292
|
+
[{
|
|
242
293
|
batchSize: { gt: 1 },
|
|
243
294
|
useGPU: true,
|
|
244
295
|
gpuSamplingAvailable: true,
|
|
245
|
-
disableMultiTokenDecode:
|
|
296
|
+
disableMultiTokenDecode: { neq: true },
|
|
246
297
|
disableCommandBatching: false,
|
|
247
298
|
isBdpaPagedLayout: false,
|
|
248
299
|
finitenessFallbackWindowOpen: false,
|
|
249
|
-
|
|
300
|
+
hasLinearAttentionLayers: false,
|
|
301
|
+
hasRangeBackedPerLayerInputs: false,
|
|
302
|
+
}, {
|
|
303
|
+
batchSize: { gt: 1 },
|
|
304
|
+
useGPU: true,
|
|
305
|
+
gpuSamplingAvailable: true,
|
|
306
|
+
disableMultiTokenDecode: { neq: true },
|
|
307
|
+
disableCommandBatching: false,
|
|
308
|
+
isBdpaPagedLayout: false,
|
|
309
|
+
finitenessFallbackWindowOpen: false,
|
|
310
|
+
hasLinearAttentionLayers: false,
|
|
311
|
+
hasRangeBackedPerLayerInputs: true,
|
|
312
|
+
selfSpeculationEnabled: false,
|
|
313
|
+
}, {
|
|
314
|
+
batchSize: { gt: 1 },
|
|
315
|
+
useGPU: true,
|
|
316
|
+
gpuSamplingAvailable: true,
|
|
317
|
+
disableMultiTokenDecode: { neq: true },
|
|
318
|
+
disableCommandBatching: false,
|
|
319
|
+
isBdpaPagedLayout: false,
|
|
320
|
+
finitenessFallbackWindowOpen: false,
|
|
321
|
+
hasLinearAttentionLayers: true,
|
|
322
|
+
hasRangeBackedPerLayerInputs: false,
|
|
323
|
+
}],
|
|
250
324
|
'batchDecodeEnabled'
|
|
251
325
|
);
|
|
252
326
|
errors.push(...batchShape.errors);
|
|
@@ -269,6 +343,56 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
269
343
|
ok: batchSemantics.ok,
|
|
270
344
|
});
|
|
271
345
|
|
|
346
|
+
const maxBatchShapeErrors = [];
|
|
347
|
+
if (!Array.isArray(maxBatchDecodeTokenRules) || maxBatchDecodeTokenRules.length !== 4) {
|
|
348
|
+
maxBatchShapeErrors.push('[ExecutionRulesContract] maxBatchDecodeTokens must contain exactly 4 rules.');
|
|
349
|
+
} else {
|
|
350
|
+
const [hotVocabularyRule, linearAttentionRule, splitTablesRule, fallbackRule] = maxBatchDecodeTokenRules;
|
|
351
|
+
if (!matchesExactObject(hotVocabularyRule?.match, { hasHotVocabularyBatchDecode: true }) || hotVocabularyRule?.value !== 1) {
|
|
352
|
+
maxBatchShapeErrors.push(
|
|
353
|
+
'[ExecutionRulesContract] maxBatchDecodeTokens hot-vocabulary rule must cap tokenizer_scores bursts at 1 token.'
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
if (!matchesExactObject(linearAttentionRule?.match, { hasLinearAttentionLayers: true }) || linearAttentionRule?.value !== 32) {
|
|
357
|
+
maxBatchShapeErrors.push(
|
|
358
|
+
'[ExecutionRulesContract] maxBatchDecodeTokens linear-attention rule must cap unsafe burst recording at 32 tokens.'
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
if (!matchesExactObject(splitTablesRule?.match, { hasGpuSplitPerLayerInputs: true }) || splitTablesRule?.value !== 4) {
|
|
362
|
+
maxBatchShapeErrors.push(
|
|
363
|
+
'[ExecutionRulesContract] maxBatchDecodeTokens enabling rule must cap gpu_split_tables bursts at 4 tokens.'
|
|
364
|
+
);
|
|
365
|
+
}
|
|
366
|
+
if (!matchesExactObject(fallbackRule?.match, {}) || fallbackRule?.value !== null) {
|
|
367
|
+
maxBatchShapeErrors.push(
|
|
368
|
+
'[ExecutionRulesContract] maxBatchDecodeTokens fallback rule must be { match: {}, value: null }.'
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
errors.push(...maxBatchShapeErrors);
|
|
373
|
+
checks.push({
|
|
374
|
+
id: 'inference.execution.maxBatchDecodeTokens.shape',
|
|
375
|
+
ok: maxBatchShapeErrors.length === 0,
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
const maxBatchSemantics = Array.isArray(maxBatchDecodeTokenRules)
|
|
379
|
+
? checkRuleSemantics(
|
|
380
|
+
maxBatchDecodeTokenRules,
|
|
381
|
+
enumerateMaxBatchDecodeTokenContexts(),
|
|
382
|
+
maxBatchDecodeTokensSemantic,
|
|
383
|
+
'maxBatchDecodeTokens'
|
|
384
|
+
)
|
|
385
|
+
: {
|
|
386
|
+
ok: false,
|
|
387
|
+
errors: ['[ExecutionRulesContract] maxBatchDecodeTokens is unavailable for semantic check.'],
|
|
388
|
+
sampledContexts: 0,
|
|
389
|
+
};
|
|
390
|
+
errors.push(...maxBatchSemantics.errors);
|
|
391
|
+
checks.push({
|
|
392
|
+
id: 'inference.execution.maxBatchDecodeTokens.semantics',
|
|
393
|
+
ok: maxBatchSemantics.ok,
|
|
394
|
+
});
|
|
395
|
+
|
|
272
396
|
return {
|
|
273
397
|
schemaVersion: 1,
|
|
274
398
|
source: 'doppler',
|
|
@@ -279,9 +403,11 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
279
403
|
decodeRecorderRules: Array.isArray(decodeRules) ? decodeRules.length : 0,
|
|
280
404
|
profileDecodeRecorderRules: Array.isArray(profileDecodeRules) ? profileDecodeRules.length : 0,
|
|
281
405
|
batchDecodeRules: Array.isArray(batchRules) ? batchRules.length : 0,
|
|
406
|
+
maxBatchDecodeTokenRules: Array.isArray(maxBatchDecodeTokenRules) ? maxBatchDecodeTokenRules.length : 0,
|
|
282
407
|
decodeRecorderContexts: decodeSemantics.sampledContexts,
|
|
283
408
|
profileDecodeRecorderContexts: profileDecodeSemantics.sampledContexts,
|
|
284
409
|
batchDecodeContexts: batchSemantics.sampledContexts,
|
|
410
|
+
maxBatchDecodeTokenContexts: maxBatchSemantics.sampledContexts,
|
|
285
411
|
},
|
|
286
412
|
};
|
|
287
413
|
}
|