doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -3,7 +3,19 @@
|
|
|
3
3
|
import { parseModelConfig } from './config.js';
|
|
4
4
|
import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
|
|
5
5
|
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
KVCache,
|
|
8
|
+
SlidingWindowKVCache,
|
|
9
|
+
TieredKVCache,
|
|
10
|
+
BasisDecomposedPagedCache,
|
|
11
|
+
QuantizedKVCache,
|
|
12
|
+
MixedGeometryKVCache,
|
|
13
|
+
} from '../../kv-cache.js';
|
|
14
|
+
import {
|
|
15
|
+
retainTurboQuantSharedBuffers,
|
|
16
|
+
} from '../../../gpu/kernels/turboquant-codebook.js';
|
|
17
|
+
import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
|
|
18
|
+
import { selectRuleValue as selectKernelRuleValue } from '../../../gpu/kernels/rule-registry.js';
|
|
7
19
|
import { Tokenizer } from '../../tokenizer.js';
|
|
8
20
|
import { MoERouter } from '../../moe-router.js';
|
|
9
21
|
import { SpeculativeDecoder } from '../../speculative.js';
|
|
@@ -11,13 +23,16 @@ import { getDopplerLoader } from '../../../loader/doppler-loader.js';
|
|
|
11
23
|
import { log, setGPUDevice, trace as debugTrace } from '../../../debug/index.js';
|
|
12
24
|
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
13
25
|
import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
|
|
14
|
-
import {
|
|
15
|
-
|
|
26
|
+
import {
|
|
27
|
+
getKernelPathMatmulVariant,
|
|
28
|
+
kernelPathRequiresF32MatmulWeights,
|
|
29
|
+
} from '../../../config/kernel-path-loader.js';
|
|
30
|
+
import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
16
31
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
17
32
|
import {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
} from '../../../
|
|
33
|
+
createHttpArtifactStorageContext,
|
|
34
|
+
createNodeFileArtifactStorageContext,
|
|
35
|
+
} from '../../../storage/artifact-storage-context.js';
|
|
21
36
|
|
|
22
37
|
function resolveErrorMessage(error) {
|
|
23
38
|
if (error && typeof error === 'object' && typeof error.message === 'string') {
|
|
@@ -29,6 +44,48 @@ function resolveErrorMessage(error) {
|
|
|
29
44
|
return String(error);
|
|
30
45
|
}
|
|
31
46
|
|
|
47
|
+
function isPlainObject(value) {
|
|
48
|
+
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isRuntimeInferenceConfigInput(value) {
|
|
52
|
+
if (!isPlainObject(value)) {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
return Object.prototype.hasOwnProperty.call(value, 'session')
|
|
56
|
+
|| Object.prototype.hasOwnProperty.call(value, 'kvcache')
|
|
57
|
+
|| Object.prototype.hasOwnProperty.call(value, 'compute')
|
|
58
|
+
|| Object.prototype.hasOwnProperty.call(value, 'batching')
|
|
59
|
+
|| Object.prototype.hasOwnProperty.call(value, 'generation')
|
|
60
|
+
|| Object.prototype.hasOwnProperty.call(value, 'kernelPath')
|
|
61
|
+
|| Object.prototype.hasOwnProperty.call(value, 'pipeline');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function resolveRuntimeKVConfig(runtimeInput) {
|
|
65
|
+
if (runtimeInput != null && !isRuntimeInferenceConfigInput(runtimeInput)) {
|
|
66
|
+
return runtimeInput;
|
|
67
|
+
}
|
|
68
|
+
const runtimeInference = runtimeInput ?? getRuntimeConfig().inference;
|
|
69
|
+
if (runtimeInference?.session?.kvcache) {
|
|
70
|
+
return runtimeInference.session.kvcache;
|
|
71
|
+
}
|
|
72
|
+
throw new Error(
|
|
73
|
+
'runtime.inference.session.kvcache is required for live KV cache creation. ' +
|
|
74
|
+
'Top-level runtime.inference.kvcache is not supported on the live path.'
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function assertSupportedTurboQuantMode(mode, label) {
|
|
79
|
+
if (String(mode ?? '').trim().toLowerCase() !== 'turboquant_outlier') {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
throw new Error(
|
|
83
|
+
`${label}="turboquant_outlier" is not supported yet. ` +
|
|
84
|
+
'TurboQuant outlier high-precision buffers and decode kernels are not wired end to end; ' +
|
|
85
|
+
'use "turboquant" or "turboquant_prod".'
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
32
89
|
function toArrayBuffer(value, label) {
|
|
33
90
|
if (value instanceof ArrayBuffer) {
|
|
34
91
|
return value;
|
|
@@ -53,95 +110,81 @@ function isRDRRManifest(manifest) {
|
|
|
53
110
|
return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
|
|
54
111
|
}
|
|
55
112
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
// This copy in text/init.js is the canonical version for inference pipeline shard loading.
|
|
59
|
-
// The duplication is intentional to avoid cross-pipeline imports; do not merge without a
|
|
60
|
-
// coordinated refactor of both pipeline families.
|
|
61
|
-
function normalizeBaseUrl(baseUrl) {
|
|
62
|
-
if (typeof baseUrl !== 'string' || baseUrl.trim().length === 0) {
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
return baseUrl.replace(/\/$/, '');
|
|
113
|
+
export function createNodeFileShardStorageContext(baseUrl, manifest) {
|
|
114
|
+
return createNodeFileArtifactStorageContext(baseUrl, manifest);
|
|
66
115
|
}
|
|
67
116
|
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
if (
|
|
71
|
-
|
|
72
|
-
const end = start + Math.max(0, Math.floor(length)) - 1;
|
|
73
|
-
headers.Range = `bytes=${start}-${end}`;
|
|
74
|
-
}
|
|
75
|
-
const response = await fetch(url, { headers });
|
|
76
|
-
if (!response.ok) {
|
|
77
|
-
throw new Error(`Failed to fetch ${url}: ${response.status}`);
|
|
117
|
+
function createRemoteStorageContext(baseUrl, manifest) {
|
|
118
|
+
const nodeFileStorageContext = createNodeFileArtifactStorageContext(baseUrl, manifest);
|
|
119
|
+
if (nodeFileStorageContext) {
|
|
120
|
+
return nodeFileStorageContext;
|
|
78
121
|
}
|
|
79
|
-
return
|
|
122
|
+
return createHttpArtifactStorageContext(baseUrl, manifest);
|
|
80
123
|
}
|
|
81
124
|
|
|
82
|
-
function createRemoteStorageContext(baseUrl, manifest) {
|
|
83
|
-
const root = normalizeBaseUrl(baseUrl);
|
|
84
|
-
if (!root || !isRDRRManifest(manifest)) {
|
|
85
|
-
return null;
|
|
86
|
-
}
|
|
87
125
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
126
|
+
/** Allowed quantizationInfo.layout values for Q4K models. */
|
|
127
|
+
const Q4K_LAYOUT_ALLOWLIST = new Set(['row', 'col']);
|
|
128
|
+
const Q4K_PROJECTION_OPS = new Set([
|
|
129
|
+
'q_proj', 'k_proj', 'v_proj', 'o_proj',
|
|
130
|
+
'gate_proj', 'up_proj', 'down_proj',
|
|
131
|
+
'qkv_proj',
|
|
132
|
+
'linear_qkv_proj',
|
|
133
|
+
'linear_z_proj',
|
|
134
|
+
'linear_a_proj',
|
|
135
|
+
'linear_b_proj',
|
|
136
|
+
'linear_out_proj',
|
|
137
|
+
]);
|
|
138
|
+
|
|
139
|
+
function summarizeQ4KProjectionKernelKinds(kernelPath) {
|
|
140
|
+
const summary = {
|
|
141
|
+
denseProjectionKernels: [],
|
|
142
|
+
fusedProjectionKernels: [],
|
|
143
|
+
};
|
|
144
|
+
if (!kernelPath || typeof kernelPath !== 'object') {
|
|
145
|
+
return summary;
|
|
93
146
|
}
|
|
94
147
|
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if (!filename) {
|
|
100
|
-
throw new Error('Direct-source artifact path is required.');
|
|
101
|
-
}
|
|
102
|
-
const url = `${root}/${filename}`;
|
|
103
|
-
return fetchBytes(url, offset, length);
|
|
104
|
-
};
|
|
105
|
-
const readText = async (relativePath) => {
|
|
106
|
-
const filename = String(relativePath || '').replace(/^\/+/, '');
|
|
107
|
-
if (!filename) return null;
|
|
108
|
-
const response = await fetch(`${root}/${filename}`);
|
|
109
|
-
if (!response.ok) {
|
|
110
|
-
throw new Error(`Failed to fetch ${filename} from ${root}: ${response.status}`);
|
|
148
|
+
const appendPhase = (phase, steps) => {
|
|
149
|
+
for (const step of steps ?? []) {
|
|
150
|
+
if (!Q4K_PROJECTION_OPS.has(step?.op)) {
|
|
151
|
+
continue;
|
|
111
152
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
const filename = String(relativePath || '').replace(/^\/+/, '');
|
|
116
|
-
if (!filename) {
|
|
117
|
-
throw new Error('Direct-source binary asset path is required.');
|
|
153
|
+
const kernel = String(step?.kernel ?? '');
|
|
154
|
+
if (!kernel) {
|
|
155
|
+
continue;
|
|
118
156
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
readText,
|
|
125
|
-
readBinary,
|
|
126
|
-
verifyHashes: true,
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
return {
|
|
131
|
-
async loadShard(index) {
|
|
132
|
-
const shard = manifest.shards[index];
|
|
133
|
-
const filename = shard?.filename;
|
|
134
|
-
if (!filename) {
|
|
135
|
-
throw new Error(`Manifest shard ${index} is missing filename.`);
|
|
157
|
+
const descriptor = `${phase}.${step.op}:${kernel}#${String(step?.entry ?? 'main')}`;
|
|
158
|
+
if (kernel.startsWith('fused_matmul_q4')) {
|
|
159
|
+
summary.fusedProjectionKernels.push(descriptor);
|
|
160
|
+
} else if (kernel.startsWith('matmul_')) {
|
|
161
|
+
summary.denseProjectionKernels.push(descriptor);
|
|
136
162
|
}
|
|
137
|
-
|
|
138
|
-
},
|
|
163
|
+
}
|
|
139
164
|
};
|
|
140
|
-
}
|
|
141
165
|
|
|
166
|
+
appendPhase('decode', kernelPath.decode?.steps);
|
|
167
|
+
appendPhase('prefill', kernelPath.prefill?.steps);
|
|
168
|
+
return summary;
|
|
169
|
+
}
|
|
142
170
|
|
|
143
|
-
|
|
144
|
-
|
|
171
|
+
function resolveQ4KProjectionMaterializationMode(
|
|
172
|
+
manifest,
|
|
173
|
+
kernelPath,
|
|
174
|
+
kernelPathSource = 'none'
|
|
175
|
+
) {
|
|
176
|
+
const summary = summarizeQ4KProjectionKernelKinds(kernelPath);
|
|
177
|
+
const mode = selectKernelRuleValue('matmul', 'q4kMaterializationMode', {
|
|
178
|
+
hasFusedProjections: summary.fusedProjectionKernels.length > 0,
|
|
179
|
+
hasDenseProjections: summary.denseProjectionKernels.length > 0,
|
|
180
|
+
});
|
|
181
|
+
debugTrace.loader(
|
|
182
|
+
`Q4K materialization: model=${manifest?.modelId ?? 'unknown'}, mode=${mode}, ` +
|
|
183
|
+
`source=${kernelPathSource}, dense=${summary.denseProjectionKernels.length}, ` +
|
|
184
|
+
`fused=${summary.fusedProjectionKernels.length}`
|
|
185
|
+
);
|
|
186
|
+
return mode;
|
|
187
|
+
}
|
|
145
188
|
|
|
146
189
|
export function resolveQ4KConfig(
|
|
147
190
|
manifest,
|
|
@@ -165,7 +208,19 @@ export function resolveQ4KConfig(
|
|
|
165
208
|
`Allowed values: ${[...Q4K_LAYOUT_ALLOWLIST].join(', ')}.`
|
|
166
209
|
);
|
|
167
210
|
}
|
|
168
|
-
|
|
211
|
+
const q4kMaterializationMode = isQ4KModel
|
|
212
|
+
? resolveQ4KProjectionMaterializationMode(manifest, kernelPath, kernelPathSource)
|
|
213
|
+
: 'dense';
|
|
214
|
+
if (isQ4KModel) {
|
|
215
|
+
debugTrace.loader(
|
|
216
|
+
`Q4K projection materialization: model=${manifest?.modelId ?? 'unknown'}, ` +
|
|
217
|
+
`mode=${q4kMaterializationMode}, source=${kernelPathSource}`
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
const hasExplicitKernelPath = kernelPath != null;
|
|
221
|
+
let useFused = hasExplicitKernelPath
|
|
222
|
+
? q4kMaterializationMode !== 'dense'
|
|
223
|
+
: hasSubgroups;
|
|
169
224
|
const kernelPathKeepsF32Weights = kernelPathRequiresF32MatmulWeights(kernelPath);
|
|
170
225
|
if (q4kLayout === 'col') {
|
|
171
226
|
useFused = false;
|
|
@@ -176,13 +231,15 @@ export function resolveQ4KConfig(
|
|
|
176
231
|
const layoutLabel = q4kLayout ?? 'none';
|
|
177
232
|
debugTrace.loader(
|
|
178
233
|
`Q4K config: fused=${useFused}, kernelPath=${pathLabel}, source=${kernelPathSource}, ` +
|
|
179
|
-
`layout=${layoutLabel},
|
|
234
|
+
`layout=${layoutLabel}, materialization=${q4kMaterializationMode}, ` +
|
|
235
|
+
`keepF32Weights=${resolvedKeepF32Weights}, subgroups=${hasSubgroups}`
|
|
180
236
|
);
|
|
181
237
|
|
|
182
238
|
return {
|
|
183
239
|
useFusedQ4K: useFused,
|
|
184
240
|
q4kLayout,
|
|
185
241
|
keepF32Weights: resolvedKeepF32Weights,
|
|
242
|
+
q4kMaterializationMode,
|
|
186
243
|
};
|
|
187
244
|
}
|
|
188
245
|
|
|
@@ -191,13 +248,21 @@ export function resolveQ4KConfig(
|
|
|
191
248
|
// ============================================================================
|
|
192
249
|
|
|
193
250
|
|
|
194
|
-
function computeRoPEFreqsForTheta(
|
|
195
|
-
|
|
251
|
+
function computeRoPEFreqsForTheta(
|
|
252
|
+
theta,
|
|
253
|
+
rotaryDim,
|
|
254
|
+
frequencyBaseDim,
|
|
255
|
+
maxSeqLen,
|
|
256
|
+
ropeScale,
|
|
257
|
+
ropeScalingType,
|
|
258
|
+
ropeScaling
|
|
259
|
+
) {
|
|
260
|
+
const halfDim = rotaryDim / 2;
|
|
196
261
|
|
|
197
262
|
// Compute base frequencies: theta_i = 1 / (base^(2i/d))
|
|
198
263
|
const freqs = new Float32Array(halfDim);
|
|
199
264
|
for (let i = 0; i < halfDim; i++) {
|
|
200
|
-
freqs[i] = 1.0 / Math.pow(theta, (2 * i) /
|
|
265
|
+
freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
|
|
201
266
|
}
|
|
202
267
|
|
|
203
268
|
// Compute per-dimension scaling factors
|
|
@@ -283,6 +348,33 @@ function isSameRoPEScalingConfig(
|
|
|
283
348
|
=== (rightScaling?.original_max_position_embeddings ?? null);
|
|
284
349
|
}
|
|
285
350
|
|
|
351
|
+
const GPU_ROPE_BUFFER_CACHE = new WeakMap();
|
|
352
|
+
const CPU_ROPE_BUFFER_CACHE = new Map();
|
|
353
|
+
|
|
354
|
+
function buildRoPECacheKey(config) {
|
|
355
|
+
return JSON.stringify({
|
|
356
|
+
headDim: config.headDim,
|
|
357
|
+
localHeadDim: config.localHeadDim ?? null,
|
|
358
|
+
rotaryDim: config.rotaryDim ?? null,
|
|
359
|
+
ropeLocalRotaryDim: config.ropeLocalRotaryDim ?? null,
|
|
360
|
+
ropeFrequencyBaseDim: config.ropeFrequencyBaseDim ?? null,
|
|
361
|
+
ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim ?? null,
|
|
362
|
+
maxSeqLen: config.maxSeqLen,
|
|
363
|
+
ropeTheta: config.ropeTheta,
|
|
364
|
+
ropeLocalTheta: config.ropeLocalTheta ?? null,
|
|
365
|
+
mropeInterleaved: config.mropeInterleaved === true,
|
|
366
|
+
mropeSection: Array.isArray(config.mropeSection) ? [...config.mropeSection] : null,
|
|
367
|
+
partialRotaryFactor: config.partialRotaryFactor ?? null,
|
|
368
|
+
ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor ?? null,
|
|
369
|
+
ropeScale: config.ropeScale,
|
|
370
|
+
ropeLocalScale: config.ropeLocalScale ?? null,
|
|
371
|
+
ropeScalingType: config.ropeScalingType ?? null,
|
|
372
|
+
ropeLocalScalingType: config.ropeLocalScalingType ?? null,
|
|
373
|
+
ropeScaling: config.ropeScaling ?? null,
|
|
374
|
+
ropeLocalScaling: config.ropeLocalScaling ?? null,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
|
|
286
378
|
function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
|
|
287
379
|
if (rotaryDim != null) {
|
|
288
380
|
if (!Number.isFinite(rotaryDim) || rotaryDim <= 0 || (rotaryDim % 2) !== 0) {
|
|
@@ -311,17 +403,37 @@ function resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor) {
|
|
|
311
403
|
return resolved;
|
|
312
404
|
}
|
|
313
405
|
|
|
406
|
+
function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
|
|
407
|
+
if (frequencyBaseDim == null) {
|
|
408
|
+
return rotaryDim;
|
|
409
|
+
}
|
|
410
|
+
if (!Number.isFinite(frequencyBaseDim) || frequencyBaseDim <= 0 || (Math.trunc(frequencyBaseDim) % 2) !== 0) {
|
|
411
|
+
throw new Error(`${label} must be a positive even integer; got "${frequencyBaseDim}".`);
|
|
412
|
+
}
|
|
413
|
+
const resolved = Math.trunc(frequencyBaseDim);
|
|
414
|
+
if (resolved < rotaryDim) {
|
|
415
|
+
throw new Error(`${label} ${resolved} cannot be smaller than rotaryDim ${rotaryDim}.`);
|
|
416
|
+
}
|
|
417
|
+
return resolved;
|
|
418
|
+
}
|
|
419
|
+
|
|
314
420
|
|
|
315
421
|
export async function initRoPEFrequencies(config, useGPU) {
|
|
422
|
+
const cacheKey = buildRoPECacheKey(config);
|
|
316
423
|
const {
|
|
317
424
|
headDim,
|
|
425
|
+
localHeadDim,
|
|
318
426
|
rotaryDim,
|
|
427
|
+
ropeLocalRotaryDim,
|
|
428
|
+
ropeFrequencyBaseDim,
|
|
429
|
+
ropeLocalFrequencyBaseDim,
|
|
319
430
|
maxSeqLen,
|
|
320
431
|
ropeTheta,
|
|
321
432
|
ropeLocalTheta,
|
|
322
433
|
mropeInterleaved,
|
|
323
434
|
mropeSection,
|
|
324
435
|
partialRotaryFactor,
|
|
436
|
+
ropeLocalPartialRotaryFactor,
|
|
325
437
|
ropeScale,
|
|
326
438
|
ropeLocalScale,
|
|
327
439
|
ropeScalingType,
|
|
@@ -347,7 +459,23 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
347
459
|
? ropeScaling
|
|
348
460
|
: ropeLocalScaling
|
|
349
461
|
);
|
|
462
|
+
const resolvedLocalHeadDim = localHeadDim ?? headDim;
|
|
350
463
|
const resolvedRotaryDim = resolveRotaryDim(headDim, rotaryDim, partialRotaryFactor);
|
|
464
|
+
const resolvedLocalRotaryDim = resolveRotaryDim(
|
|
465
|
+
resolvedLocalHeadDim,
|
|
466
|
+
ropeLocalRotaryDim,
|
|
467
|
+
ropeLocalPartialRotaryFactor
|
|
468
|
+
);
|
|
469
|
+
const resolvedFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
470
|
+
resolvedRotaryDim,
|
|
471
|
+
ropeFrequencyBaseDim,
|
|
472
|
+
'RoPE frequency base dim'
|
|
473
|
+
);
|
|
474
|
+
const resolvedLocalFrequencyBaseDim = resolveFrequencyBaseDim(
|
|
475
|
+
resolvedLocalRotaryDim,
|
|
476
|
+
ropeLocalFrequencyBaseDim,
|
|
477
|
+
'Local RoPE frequency base dim'
|
|
478
|
+
);
|
|
351
479
|
const halfDim = resolvedRotaryDim / 2;
|
|
352
480
|
if (mropeInterleaved === true && Array.isArray(mropeSection)) {
|
|
353
481
|
const expandedDim = mropeSection.reduce((sum, entry) => sum + entry, 0) * 2;
|
|
@@ -363,7 +491,13 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
363
491
|
|
|
364
492
|
// Compute global (full_attention) frequencies
|
|
365
493
|
const globalFreqs = computeRoPEFreqsForTheta(
|
|
366
|
-
ropeTheta,
|
|
494
|
+
ropeTheta,
|
|
495
|
+
resolvedRotaryDim,
|
|
496
|
+
resolvedFrequencyBaseDim,
|
|
497
|
+
maxSeqLen,
|
|
498
|
+
ropeScale,
|
|
499
|
+
ropeScalingType,
|
|
500
|
+
ropeScaling
|
|
367
501
|
);
|
|
368
502
|
|
|
369
503
|
// Compute local (sliding_attention) frequencies if different from global.
|
|
@@ -371,6 +505,7 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
371
505
|
|
|
372
506
|
let localFreqs = null;
|
|
373
507
|
const hasDistinctLocalTheta = resolvedLocalTheta !== ropeTheta;
|
|
508
|
+
const hasDistinctLocalDim = resolvedLocalRotaryDim !== resolvedRotaryDim;
|
|
374
509
|
const hasDistinctLocalScaling = !isSameRoPEScalingConfig(
|
|
375
510
|
ropeScalingType,
|
|
376
511
|
ropeScale,
|
|
@@ -379,10 +514,11 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
379
514
|
resolvedLocalScale,
|
|
380
515
|
resolvedLocalScaling
|
|
381
516
|
);
|
|
382
|
-
if (hasDistinctLocalTheta || hasDistinctLocalScaling) {
|
|
517
|
+
if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
|
|
383
518
|
localFreqs = computeRoPEFreqsForTheta(
|
|
384
519
|
resolvedLocalTheta,
|
|
385
|
-
|
|
520
|
+
resolvedLocalRotaryDim,
|
|
521
|
+
resolvedLocalFrequencyBaseDim,
|
|
386
522
|
maxSeqLen,
|
|
387
523
|
resolvedLocalScale,
|
|
388
524
|
resolvedLocalScalingType,
|
|
@@ -391,6 +527,8 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
391
527
|
log.debug(
|
|
392
528
|
'Pipeline',
|
|
393
529
|
`Dual RoPE: local theta=${resolvedLocalTheta}, global theta=${ropeTheta}, ` +
|
|
530
|
+
`localRotaryDim=${resolvedLocalRotaryDim}, globalRotaryDim=${resolvedRotaryDim}, ` +
|
|
531
|
+
`localFrequencyBaseDim=${resolvedLocalFrequencyBaseDim}, globalFrequencyBaseDim=${resolvedFrequencyBaseDim}, ` +
|
|
394
532
|
`localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}, ` +
|
|
395
533
|
`globalScaling=${ropeScalingType ?? 'none'}:${ropeScale}`
|
|
396
534
|
);
|
|
@@ -411,6 +549,15 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
411
549
|
// Upload to GPU if available
|
|
412
550
|
const device = getDevice();
|
|
413
551
|
if (device && useGPU) {
|
|
552
|
+
let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
|
|
553
|
+
if (!perDeviceCache) {
|
|
554
|
+
perDeviceCache = new Map();
|
|
555
|
+
GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
|
|
556
|
+
}
|
|
557
|
+
const cachedBuffers = perDeviceCache.get(cacheKey);
|
|
558
|
+
if (cachedBuffers) {
|
|
559
|
+
return cachedBuffers;
|
|
560
|
+
}
|
|
414
561
|
let cosBuffer = null;
|
|
415
562
|
let sinBuffer = null;
|
|
416
563
|
let localCosBuffer = null;
|
|
@@ -440,38 +587,49 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
440
587
|
'Pipeline',
|
|
441
588
|
`RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
|
|
442
589
|
`theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
|
|
590
|
+
`${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
|
|
443
591
|
`scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
|
|
444
592
|
`interleaved=${mropeInterleaved === true}`
|
|
445
593
|
);
|
|
446
594
|
|
|
447
|
-
|
|
595
|
+
const buffers = {
|
|
448
596
|
cos: cosBuffer,
|
|
449
597
|
sin: sinBuffer,
|
|
450
598
|
localCos: localCosBuffer,
|
|
451
599
|
localSin: localSinBuffer,
|
|
452
600
|
};
|
|
601
|
+
perDeviceCache.set(cacheKey, buffers);
|
|
602
|
+
return buffers;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
|
|
606
|
+
if (cachedCpuBuffers) {
|
|
607
|
+
return cachedCpuBuffers;
|
|
453
608
|
}
|
|
454
609
|
|
|
455
610
|
log.debug(
|
|
456
611
|
'Pipeline',
|
|
457
612
|
`RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
|
|
458
613
|
`theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
|
|
614
|
+
`${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
|
|
459
615
|
`scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
|
|
460
616
|
`interleaved=${mropeInterleaved === true}`
|
|
461
617
|
);
|
|
462
618
|
|
|
463
|
-
|
|
619
|
+
const buffers = {
|
|
464
620
|
cos: globalFreqs.cos,
|
|
465
621
|
sin: globalFreqs.sin,
|
|
466
622
|
localCos: localFreqs?.cos,
|
|
467
623
|
localSin: localFreqs?.sin,
|
|
468
624
|
};
|
|
625
|
+
CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
|
|
626
|
+
return buffers;
|
|
469
627
|
}
|
|
470
628
|
|
|
471
629
|
|
|
472
630
|
export function isGPURoPEBuffers(buffers) {
|
|
473
631
|
if (typeof GPUBuffer === 'undefined') return false;
|
|
474
|
-
return !!buffers?.cos && buffers.cos
|
|
632
|
+
return !!buffers?.cos && isGpuBufferInstance(buffers.cos);
|
|
475
633
|
}
|
|
476
634
|
|
|
477
635
|
|
|
@@ -494,6 +652,128 @@ function hasFullAttentionLayers(layerTypes) {
|
|
|
494
652
|
return layerTypes.some((layerType) => !isSlidingLayerType(layerType));
|
|
495
653
|
}
|
|
496
654
|
|
|
655
|
+
function resolveContiguousKVPolicy(modelConfig) {
|
|
656
|
+
if (hasFullAttentionLayers(modelConfig.layerTypes)) {
|
|
657
|
+
return {
|
|
658
|
+
forceContiguousKVCache: true,
|
|
659
|
+
reason: 'layerPattern',
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
if (modelConfig.layerTypes == null && modelConfig.slidingWindow == null) {
|
|
663
|
+
return {
|
|
664
|
+
forceContiguousKVCache: true,
|
|
665
|
+
reason: 'slidingWindow',
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
return {
|
|
669
|
+
forceContiguousKVCache: false,
|
|
670
|
+
reason: 'none',
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function usesMixedGeometryKVCache(modelConfig) {
|
|
675
|
+
if (!modelConfig || typeof modelConfig !== 'object') {
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
const hasMixedAttentionGeometry = Number.isFinite(modelConfig.globalHeadDim)
|
|
679
|
+
&& modelConfig.globalHeadDim > 0
|
|
680
|
+
&& modelConfig.globalHeadDim !== modelConfig.headDim;
|
|
681
|
+
const hasSharedKvLayers = Number.isFinite(modelConfig.numKvSharedLayers)
|
|
682
|
+
&& modelConfig.numKvSharedLayers > 0;
|
|
683
|
+
const hasExplicitLayerTypes = Array.isArray(modelConfig.layerTypes)
|
|
684
|
+
&& modelConfig.layerTypes.length === modelConfig.numLayers;
|
|
685
|
+
return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
function getRequiredKernelMaxKVLen(operation, variant, label) {
|
|
689
|
+
const config = getKernelConfig(operation, variant);
|
|
690
|
+
const maxKVLen = config.variantMetadata?.maxKVLen;
|
|
691
|
+
if (!Number.isFinite(maxKVLen) || maxKVLen <= 0) {
|
|
692
|
+
throw new Error(`${label} kernel "${variant}" is missing variantMetadata.maxKVLen.`);
|
|
693
|
+
}
|
|
694
|
+
return maxKVLen;
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
function resolveTieredRequestedQuantMode(runtimeKV) {
|
|
698
|
+
const tieringMode = String(runtimeKV?.tiering?.mode ?? 'off').trim().toLowerCase();
|
|
699
|
+
assertSupportedTurboQuantMode(tieringMode, 'runtime.inference.session.kvcache.tiering.mode');
|
|
700
|
+
const compressionMode = String(
|
|
701
|
+
runtimeKV?.tiering?.compression?.mode
|
|
702
|
+
?? (tieringMode === 'int8'
|
|
703
|
+
|| tieringMode === 'int4'
|
|
704
|
+
|| tieringMode === 'turboquant'
|
|
705
|
+
|| tieringMode === 'turboquant_prod'
|
|
706
|
+
? tieringMode
|
|
707
|
+
: 'none')
|
|
708
|
+
).trim().toLowerCase();
|
|
709
|
+
assertSupportedTurboQuantMode(
|
|
710
|
+
compressionMode,
|
|
711
|
+
'runtime.inference.session.kvcache.tiering.compression.mode'
|
|
712
|
+
);
|
|
713
|
+
const gatingMode = String(runtimeKV?.tiering?.gating?.mode ?? 'auto').trim().toLowerCase();
|
|
714
|
+
if (gatingMode === 'force_off') {
|
|
715
|
+
return 'none';
|
|
716
|
+
}
|
|
717
|
+
return compressionMode;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function resolveContiguousRequestedQuantMode(runtimeKV) {
|
|
721
|
+
const quantMode = String(runtimeKV?.quantization?.mode ?? 'none').trim().toLowerCase();
|
|
722
|
+
assertSupportedTurboQuantMode(
|
|
723
|
+
quantMode,
|
|
724
|
+
'runtime.inference.session.kvcache.quantization.mode'
|
|
725
|
+
);
|
|
726
|
+
return quantMode;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV) {
|
|
730
|
+
if (cacheLayout === 'contiguous_quantized') {
|
|
731
|
+
if (modelConfig.headDim > 256) {
|
|
732
|
+
throw new Error(
|
|
733
|
+
`Contiguous quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
|
|
737
|
+
const variant = selectKernelRuleValue('attention', 'contiguousQuantVariant', { mode: quantMode });
|
|
738
|
+
const maxKVLen = getRequiredKernelMaxKVLen(
|
|
739
|
+
'attention_contiguous_quant',
|
|
740
|
+
variant,
|
|
741
|
+
'Contiguous quant attention'
|
|
742
|
+
);
|
|
743
|
+
if (cacheMaxSeqLen > maxKVLen) {
|
|
744
|
+
throw new Error(
|
|
745
|
+
`Contiguous quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
|
|
746
|
+
);
|
|
747
|
+
}
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
if (cacheLayout !== 'tiered') {
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const coldQuantMode = resolveTieredRequestedQuantMode(runtimeKV);
|
|
756
|
+
if (coldQuantMode === 'none') {
|
|
757
|
+
return;
|
|
758
|
+
}
|
|
759
|
+
if (modelConfig.headDim > 256) {
|
|
760
|
+
throw new Error(
|
|
761
|
+
`Tiered quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
const variant = selectKernelRuleValue('attention', 'tieredQuantVariant', { mode: coldQuantMode });
|
|
765
|
+
const maxKVLen = getRequiredKernelMaxKVLen(
|
|
766
|
+
'attention_tiered_quant',
|
|
767
|
+
variant,
|
|
768
|
+
'Tiered quant attention'
|
|
769
|
+
);
|
|
770
|
+
if (cacheMaxSeqLen > maxKVLen) {
|
|
771
|
+
throw new Error(
|
|
772
|
+
`Tiered quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
|
|
773
|
+
);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
|
|
497
777
|
|
|
498
778
|
// ============================================================================
|
|
499
779
|
// KV Cache Setup
|
|
@@ -501,8 +781,16 @@ function hasFullAttentionLayers(layerTypes) {
|
|
|
501
781
|
|
|
502
782
|
|
|
503
783
|
export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig) {
|
|
504
|
-
|
|
505
|
-
|
|
784
|
+
if (modelConfig?.decodeStrategy === 'replay_prefill') {
|
|
785
|
+
throw new Error(
|
|
786
|
+
'Live KV cache creation is not supported for models that require replay-prefill decode. ' +
|
|
787
|
+
'Skip createKVCache() when the model config does not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
|
|
788
|
+
);
|
|
789
|
+
}
|
|
790
|
+
const runtimeKV = resolveRuntimeKVConfig(runtimeConfig);
|
|
791
|
+
const requiresMixedGeometryKVCache = usesMixedGeometryKVCache(modelConfig);
|
|
792
|
+
const contiguousKVPolicy = resolveContiguousKVPolicy(modelConfig);
|
|
793
|
+
const forceContiguousKVCache = contiguousKVPolicy.forceContiguousKVCache;
|
|
506
794
|
const modelMaxSeqLen = modelConfig.maxSeqLen;
|
|
507
795
|
if (!Number.isFinite(modelMaxSeqLen) || modelMaxSeqLen <= 0) {
|
|
508
796
|
throw new Error('Model config is missing maxSeqLen.');
|
|
@@ -517,19 +805,19 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
517
805
|
|
|
518
806
|
let cacheLayout = runtimeKV.layout;
|
|
519
807
|
if (!cacheLayout) {
|
|
520
|
-
throw new Error('runtime.inference.kvcache.layout is required.');
|
|
808
|
+
throw new Error('runtime.inference.session.kvcache.layout is required.');
|
|
521
809
|
}
|
|
522
810
|
if (cacheLayout === 'tiered' && !runtimeKV.tiering) {
|
|
523
|
-
throw new Error('runtime.inference.kvcache.tiering is required for tiered layout.');
|
|
811
|
+
throw new Error('runtime.inference.session.kvcache.tiering is required for tiered layout.');
|
|
524
812
|
}
|
|
525
813
|
const tieringMode = runtimeKV.tiering?.mode;
|
|
526
814
|
if (tieringMode == null) {
|
|
527
|
-
throw new Error('runtime.inference.kvcache.tiering.mode is required.');
|
|
815
|
+
throw new Error('runtime.inference.session.kvcache.tiering.mode is required.');
|
|
528
816
|
}
|
|
529
817
|
let layoutSource = 'runtime';
|
|
530
818
|
if (tieringMode !== 'off' && cacheLayout !== 'tiered') {
|
|
531
819
|
if (cacheLayout !== 'contiguous') {
|
|
532
|
-
throw new Error('runtime.inference.kvcache.layout must be "tiered" when tiering.mode is enabled.');
|
|
820
|
+
throw new Error('runtime.inference.session.kvcache.layout must be "tiered" when tiering.mode is enabled.');
|
|
533
821
|
}
|
|
534
822
|
cacheLayout = 'tiered';
|
|
535
823
|
layoutSource = 'tiering';
|
|
@@ -538,12 +826,42 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
538
826
|
cacheLayout = 'paged';
|
|
539
827
|
layoutSource = 'threshold';
|
|
540
828
|
}
|
|
829
|
+
const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
|
|
830
|
+
if (forceContiguousKVCache && cacheLayout === 'contiguous' && quantMode !== 'none') {
|
|
831
|
+
cacheLayout = 'contiguous_quantized';
|
|
832
|
+
layoutSource = 'quantization';
|
|
833
|
+
}
|
|
541
834
|
if (forceContiguousKVCache && cacheLayout === 'paged') {
|
|
542
835
|
throw new Error(
|
|
543
836
|
'Paged KV cache layout is not supported for models with full-attention layers. ' +
|
|
544
|
-
'Set runtime.inference.kvcache.layout to "contiguous" instead.'
|
|
837
|
+
'Set runtime.inference.session.kvcache.layout to "contiguous" instead.'
|
|
545
838
|
);
|
|
546
839
|
}
|
|
840
|
+
if (requiresMixedGeometryKVCache) {
|
|
841
|
+
if (cacheLayout !== 'contiguous') {
|
|
842
|
+
throw new Error(
|
|
843
|
+
`Mixed-geometry incremental KV cache requires layout="contiguous"; got "${cacheLayout}". ` +
|
|
844
|
+
'Disable tiering and quantized KV cache for this model.'
|
|
845
|
+
);
|
|
846
|
+
}
|
|
847
|
+
if (runtimeKV.tiering?.mode != null && runtimeKV.tiering.mode !== 'off') {
|
|
848
|
+
throw new Error(
|
|
849
|
+
`Mixed-geometry incremental KV cache requires tiering.mode="off"; got "${runtimeKV.tiering.mode}".`
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
if (resolveContiguousRequestedQuantMode(runtimeKV) !== 'none') {
|
|
853
|
+
throw new Error(
|
|
854
|
+
'Mixed-geometry incremental KV cache does not support contiguous quantization yet. ' +
|
|
855
|
+
'Set runtime.inference.session.kvcache.quantization.mode="none".'
|
|
856
|
+
);
|
|
857
|
+
}
|
|
858
|
+
if (!useGPU) {
|
|
859
|
+
throw new Error(
|
|
860
|
+
'Mixed-geometry incremental KV cache requires GPU execution. ' +
|
|
861
|
+
'Use a WebGPU-capable surface.'
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
547
865
|
if (debug && cacheLayout !== runtimeKV.layout) {
|
|
548
866
|
log.debug('Pipeline', `KV cache layout override: ${runtimeKV.layout} -> ${cacheLayout} (${layoutSource})`);
|
|
549
867
|
}
|
|
@@ -581,11 +899,23 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
581
899
|
if (cacheLayout === 'tiered' && kvDtype !== 'f16') {
|
|
582
900
|
throw new Error('Tiered KV cache requires kvDtype="f16" (no f32 tiered kernels yet).');
|
|
583
901
|
}
|
|
902
|
+
if (cacheLayout === 'contiguous_quantized' && kvDtype !== 'f16') {
|
|
903
|
+
throw new Error('Contiguous quantized KV cache requires kvDtype="f16".');
|
|
904
|
+
}
|
|
905
|
+
if (cacheLayout === 'contiguous_quantized' && !useGPU) {
|
|
906
|
+
throw new Error('Contiguous quantized KV cache requires GPU.');
|
|
907
|
+
}
|
|
584
908
|
|
|
585
|
-
if (useGPU && (cacheLayout === 'paged' || cacheLayout === 'tiered' || cacheLayout === 'bdpa')) {
|
|
909
|
+
if (useGPU && (cacheLayout === 'paged' || cacheLayout === 'tiered' || cacheLayout === 'bdpa' || cacheLayout === 'contiguous_quantized')) {
|
|
586
910
|
const limits = getDeviceLimits();
|
|
587
911
|
if (limits) {
|
|
588
|
-
const
|
|
912
|
+
const maxLayerHeadDim = Math.max(
|
|
913
|
+
modelConfig.headDim,
|
|
914
|
+
Number.isFinite(modelConfig.globalHeadDim) && modelConfig.globalHeadDim > 0
|
|
915
|
+
? modelConfig.globalHeadDim
|
|
916
|
+
: modelConfig.headDim
|
|
917
|
+
);
|
|
918
|
+
const bytesPerToken = modelConfig.numKVHeads * maxLayerHeadDim * (kvDtype === 'f16' ? 2 : 4);
|
|
589
919
|
const maxByBinding = Math.floor(limits.maxStorageBufferBindingSize / bytesPerToken);
|
|
590
920
|
const maxByBuffer = Math.floor(limits.maxBufferSize / bytesPerToken);
|
|
591
921
|
const fallbackMax = Number.isFinite(runtimeKV.gpuPagedFallbackMaxSeqLen) && runtimeKV.gpuPagedFallbackMaxSeqLen > 0
|
|
@@ -605,6 +935,8 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
605
935
|
}
|
|
606
936
|
}
|
|
607
937
|
|
|
938
|
+
assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV);
|
|
939
|
+
|
|
608
940
|
|
|
609
941
|
const cacheConfig = {
|
|
610
942
|
numLayers: modelConfig.numLayers,
|
|
@@ -621,7 +953,15 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
621
953
|
|
|
622
954
|
let kvCache;
|
|
623
955
|
|
|
624
|
-
if (
|
|
956
|
+
if (requiresMixedGeometryKVCache) {
|
|
957
|
+
kvCache = new MixedGeometryKVCache({
|
|
958
|
+
...cacheConfig,
|
|
959
|
+
numHeads: modelConfig.numKVHeads,
|
|
960
|
+
globalHeadDim: modelConfig.globalHeadDim ?? null,
|
|
961
|
+
slidingWindow,
|
|
962
|
+
layerTypes: modelConfig.layerTypes,
|
|
963
|
+
});
|
|
964
|
+
} else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
|
|
625
965
|
kvCache = new SlidingWindowKVCache({
|
|
626
966
|
...cacheConfig,
|
|
627
967
|
windowSize: slidingWindow ?? modelConfig.slidingWindow,
|
|
@@ -635,6 +975,21 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
635
975
|
...cacheConfig,
|
|
636
976
|
tiering: runtimeKV.tiering,
|
|
637
977
|
});
|
|
978
|
+
} else if (cacheLayout === 'contiguous_quantized') {
|
|
979
|
+
const quantCfg = runtimeKV.quantization ?? {};
|
|
980
|
+
const qCache = new QuantizedKVCache({
|
|
981
|
+
...cacheConfig,
|
|
982
|
+
quantMode,
|
|
983
|
+
bitWidth: quantCfg.bitWidth ?? 4,
|
|
984
|
+
prodMode: quantCfg.prodMode === true,
|
|
985
|
+
});
|
|
986
|
+
const device = getDevice();
|
|
987
|
+
qCache.setSharedBuffers(retainTurboQuantSharedBuffers(device, {
|
|
988
|
+
headDim: modelConfig.headDim,
|
|
989
|
+
bitWidth: quantCfg.bitWidth ?? 4,
|
|
990
|
+
prodMode: quantCfg.prodMode === true,
|
|
991
|
+
}));
|
|
992
|
+
kvCache = qCache;
|
|
638
993
|
} else {
|
|
639
994
|
kvCache = new KVCache(cacheConfig);
|
|
640
995
|
}
|
|
@@ -667,8 +1022,10 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
667
1022
|
);
|
|
668
1023
|
|
|
669
1024
|
if (debug) {
|
|
670
|
-
if (
|
|
1025
|
+
if (contiguousKVPolicy.reason === 'layerPattern') {
|
|
671
1026
|
log.debug('Pipeline', 'Layer pattern includes full-attention layers; paged layout blocked, contiguous enforced.');
|
|
1027
|
+
} else if (contiguousKVPolicy.reason === 'slidingWindow') {
|
|
1028
|
+
log.debug('Pipeline', 'Model declares attention.slidingWindow=null without explicit layerTypes; treating KV layout as contiguous-compatible.');
|
|
672
1029
|
}
|
|
673
1030
|
const isSliding = kvCache instanceof SlidingWindowKVCache;
|
|
674
1031
|
log.debug('Pipeline', `KV cache: type=${kvCache?.constructor?.name || 'unknown'}, kvDtype=${kvCache.kvDtype}, layout=${kvCache.layout}, maxSeqLen=${kvCache.maxSeqLen}, windowSize=${isSliding ? kvCache.windowSize : null}`);
|
|
@@ -709,6 +1066,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
709
1066
|
loadingConfig,
|
|
710
1067
|
baseUrl,
|
|
711
1068
|
loaderDebug,
|
|
1069
|
+
perLayerInputSession,
|
|
712
1070
|
} = options;
|
|
713
1071
|
const runtimeStorageContext = options.storageContext
|
|
714
1072
|
?? createRemoteStorageContext(baseUrl, manifest);
|
|
@@ -732,6 +1090,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
732
1090
|
)
|
|
733
1091
|
);
|
|
734
1092
|
dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
|
|
1093
|
+
dopplerLoader.setPerLayerInputSession(perLayerInputSession ?? modelConfig.perLayerInputsSession);
|
|
735
1094
|
|
|
736
1095
|
const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
|
|
737
1096
|
if (baseUrl && tensorsFile) {
|
|
@@ -832,6 +1191,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
832
1191
|
lmHead: dopplerLoader.lmHead,
|
|
833
1192
|
finalNorm: dopplerLoader.finalNorm,
|
|
834
1193
|
embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
|
|
1194
|
+
perLayerInputWeights: dopplerLoader.perLayerInputWeights,
|
|
835
1195
|
layerRouterWeights,
|
|
836
1196
|
};
|
|
837
1197
|
}
|
|
@@ -846,7 +1206,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
846
1206
|
function applyTurnBasedTemplate(prompt) {
|
|
847
1207
|
// Turn-based format: <start_of_turn>role\ncontent<end_of_turn>
|
|
848
1208
|
const userTurn = `<start_of_turn>user\n${prompt}<end_of_turn>\n`;
|
|
849
|
-
const modelTurn =
|
|
1209
|
+
const modelTurn = '<start_of_turn>model\n';
|
|
850
1210
|
return userTurn + modelTurn;
|
|
851
1211
|
}
|
|
852
1212
|
|
|
@@ -865,8 +1225,18 @@ function applyChatMLTemplate(prompt) {
|
|
|
865
1225
|
return `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
|
|
866
1226
|
}
|
|
867
1227
|
|
|
868
|
-
function
|
|
869
|
-
return
|
|
1228
|
+
function getQwenAssistantPrefix(options) {
|
|
1229
|
+
return options?.thinking === true
|
|
1230
|
+
? '<|im_start|>assistant\n<think>\n'
|
|
1231
|
+
: '<|im_start|>assistant\n<think>\n\n</think>\n\n';
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
function applyQwenTemplate(prompt, options) {
|
|
1235
|
+
return `<|im_start|>user\n${prompt}<|im_end|>\n${getQwenAssistantPrefix(options)}`;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
function applyGemma4Template(prompt) {
|
|
1239
|
+
return `<bos><|turn>user\n${prompt}<turn|>\n<|turn>model\n`;
|
|
870
1240
|
}
|
|
871
1241
|
|
|
872
1242
|
function applyTranslateGemmaTemplate() {
|
|
@@ -880,6 +1250,7 @@ function applyTranslateGemmaTemplate() {
|
|
|
880
1250
|
// Add new template types here rather than adding switch cases.
|
|
881
1251
|
const PROMPT_TEMPLATES = {
|
|
882
1252
|
'gemma': applyTurnBasedTemplate,
|
|
1253
|
+
'gemma4': applyGemma4Template,
|
|
883
1254
|
'llama3': applyHeaderBasedTemplate,
|
|
884
1255
|
'gpt-oss': applyChannelBasedTemplate,
|
|
885
1256
|
'chatml': applyChatMLTemplate,
|
|
@@ -887,19 +1258,20 @@ const PROMPT_TEMPLATES = {
|
|
|
887
1258
|
'translategemma': applyTranslateGemmaTemplate,
|
|
888
1259
|
};
|
|
889
1260
|
|
|
890
|
-
export function applyChatTemplate(prompt, templateType) {
|
|
1261
|
+
export function applyChatTemplate(prompt, templateType, options = undefined) {
|
|
891
1262
|
if (templateType == null) {
|
|
892
1263
|
return prompt;
|
|
893
1264
|
}
|
|
894
1265
|
const formatter = PROMPT_TEMPLATES[templateType];
|
|
895
1266
|
if (formatter) {
|
|
896
|
-
return formatter(prompt);
|
|
1267
|
+
return formatter(prompt, options);
|
|
897
1268
|
}
|
|
898
1269
|
throw new Error(`Unrecognized chat template type: ${templateType}`);
|
|
899
1270
|
}
|
|
900
1271
|
|
|
901
1272
|
// Exports preserved for existing external imports.
|
|
902
1273
|
export const applyGemmaChatTemplate = applyTurnBasedTemplate;
|
|
1274
|
+
export const applyGemma4ChatTemplate = applyGemma4Template;
|
|
903
1275
|
export const applyLlama3ChatTemplate = applyHeaderBasedTemplate;
|
|
904
1276
|
export const applyGptOssChatTemplate = applyChannelBasedTemplate;
|
|
905
1277
|
export const applyQwenChatTemplate = applyQwenTemplate;
|
|
@@ -974,12 +1346,48 @@ export function initSpeculativeDecoder(manifest, speculativeConfig) {
|
|
|
974
1346
|
// ============================================================================
|
|
975
1347
|
|
|
976
1348
|
|
|
977
|
-
|
|
1349
|
+
function kernelPathVariantRequiresQ4KWeights(variant) {
|
|
1350
|
+
if (typeof variant !== 'string' || variant.length === 0) {
|
|
1351
|
+
return false;
|
|
1352
|
+
}
|
|
1353
|
+
let config;
|
|
1354
|
+
try {
|
|
1355
|
+
config = getKernelConfig('matmul', variant);
|
|
1356
|
+
} catch {
|
|
1357
|
+
return false;
|
|
1358
|
+
}
|
|
1359
|
+
const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
|
|
1360
|
+
return shaderFile.startsWith('fused_matmul_q4');
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
|
|
1364
|
+
if (!kernelPath) {
|
|
1365
|
+
return false;
|
|
1366
|
+
}
|
|
1367
|
+
const prefillVariant = getKernelPathMatmulVariant('qkv_proj', 'prefill', layerIdx, kernelPath);
|
|
1368
|
+
if (kernelPathVariantRequiresQ4KWeights(prefillVariant)) {
|
|
1369
|
+
return true;
|
|
1370
|
+
}
|
|
1371
|
+
const decodeVariant = getKernelPathMatmulVariant('qkv_proj', 'decode', layerIdx, kernelPath);
|
|
1372
|
+
return kernelPathVariantRequiresQ4KWeights(decodeVariant);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
|
|
978
1376
|
const device = getDevice();
|
|
979
1377
|
if (!device) {
|
|
980
1378
|
log.debug('QKV Fusion', 'No GPU device, skipping fusion');
|
|
981
1379
|
return;
|
|
982
1380
|
}
|
|
1381
|
+
if (
|
|
1382
|
+
Number.isFinite(modelConfig?.globalHeadDim)
|
|
1383
|
+
&& modelConfig.globalHeadDim !== modelConfig.headDim
|
|
1384
|
+
) {
|
|
1385
|
+
log.debug(
|
|
1386
|
+
'QKV Fusion',
|
|
1387
|
+
'Skipping QKV fusion for mixed-head-dim model; per-layer attention geometry must stay explicit.'
|
|
1388
|
+
);
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
983
1391
|
|
|
984
1392
|
const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
|
|
985
1393
|
const qSize = numHeads * headDim;
|
|
@@ -996,7 +1404,7 @@ export function fuseQKVWeights(layerWeights, modelConfig) {
|
|
|
996
1404
|
shape: Array.isArray(value.shape) ? value.shape : null,
|
|
997
1405
|
};
|
|
998
1406
|
}
|
|
999
|
-
if (value
|
|
1407
|
+
if (isGpuBufferInstance(value)) {
|
|
1000
1408
|
return {
|
|
1001
1409
|
buffer: value,
|
|
1002
1410
|
dtype: getWeightDtype(value),
|
|
@@ -1029,6 +1437,13 @@ export function fuseQKVWeights(layerWeights, modelConfig) {
|
|
|
1029
1437
|
|
|
1030
1438
|
// Skip if already fused or if weights are not GPUBuffers
|
|
1031
1439
|
if (weights.qkvProj) continue;
|
|
1440
|
+
if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
|
|
1441
|
+
log.debug(
|
|
1442
|
+
'QKV Fusion',
|
|
1443
|
+
`Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
|
|
1444
|
+
);
|
|
1445
|
+
continue;
|
|
1446
|
+
}
|
|
1032
1447
|
const qProj = resolveWeight(weights.qProj);
|
|
1033
1448
|
const kProj = resolveWeight(weights.kProj);
|
|
1034
1449
|
const vProj = resolveWeight(weights.vProj);
|