doppler-gpu 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -1
- package/README.md +97 -29
- package/package.json +40 -27
- package/src/cli/config/doppler-quickstart.json +10 -0
- package/src/cli/doppler-cli.js +66 -22
- package/src/cli/doppler-quickstart.d.ts +61 -0
- package/src/cli/doppler-quickstart.js +299 -0
- package/src/cli/doppler-serve.js +395 -0
- package/src/client/doppler-api.browser.js +13 -377
- package/src/client/doppler-api.d.ts +35 -56
- package/src/client/doppler-api.js +12 -363
- package/src/client/doppler-registry.json +55 -2
- package/src/client/runtime/index.d.ts +63 -0
- package/src/client/runtime/index.js +173 -0
- package/src/client/runtime/lora.d.ts +36 -0
- package/src/client/runtime/lora.js +165 -0
- package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
- package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
- package/src/client/runtime/model-session.d.ts +72 -0
- package/src/client/runtime/model-session.js +122 -0
- package/src/client/runtime/model-source.d.ts +43 -0
- package/src/client/runtime/model-source.js +101 -0
- package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
- package/src/client/runtime/source-runtime.js +919 -0
- package/src/client/runtime/storage.d.ts +5 -0
- package/src/client/runtime/storage.js +51 -0
- package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
- package/src/config/README.md +2 -0
- package/src/config/conversion/README.md +47 -7
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
- package/src/config/execution-contract-check.d.ts +2 -1
- package/src/config/execution-contract-check.js +187 -53
- package/src/config/kernel-path-loader.d.ts +36 -0
- package/src/config/kernel-path-loader.js +97 -10
- package/src/config/kernels/kernel-ref-digests.js +40 -24
- package/src/config/kernels/registry.js +0 -1
- package/src/config/kernels/registry.json +772 -66
- package/src/config/merge-contract-check.js +3 -0
- package/src/config/merge.d.ts +18 -1
- package/src/config/merge.js +56 -0
- package/src/config/param-categories.js +5 -0
- package/src/config/param-validator.js +42 -26
- package/src/config/platforms/apple-m3.json +1 -1
- package/src/config/platforms/loader.js +21 -16
- package/src/config/required-inference-fields-contract-check.js +16 -0
- package/src/config/runtime/compute/f16-batched.json +2 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
- package/src/config/runtime/profiles/turboquant.json +36 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -0
- package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
- package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
- package/src/config/runtime.js +11 -4
- package/src/config/schema/converter.schema.d.ts +4 -2
- package/src/config/schema/converter.schema.js +7 -5
- package/src/config/schema/doppler.schema.d.ts +26 -2
- package/src/config/schema/doppler.schema.js +77 -28
- package/src/config/schema/execution-v1.schema.d.ts +78 -5
- package/src/config/schema/execution-v1.schema.js +62 -1
- package/src/config/schema/index.d.ts +22 -29
- package/src/config/schema/index.js +15 -17
- package/src/config/schema/inference.schema.d.ts +26 -4
- package/src/config/schema/kernel-path.schema.d.ts +30 -3
- package/src/config/schema/kernel-registry.schema.d.ts +1 -1
- package/src/config/schema/kernel-registry.schema.js +1 -0
- package/src/config/schema/kvcache.schema.js +7 -0
- package/src/config/schema/loading.schema.d.ts +28 -0
- package/src/config/schema/loading.schema.js +9 -0
- package/src/config/schema/manifest.schema.d.ts +79 -5
- package/src/config/schema/manifest.schema.js +7 -1
- package/src/config/schema/platform.schema.d.ts +10 -8
- package/src/config/schema/speculation-self.schema.js +0 -5
- package/src/config/schema/support-tiers.schema.d.ts +53 -0
- package/src/config/schema/support-tiers.schema.js +128 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
- package/src/config/source-packages/registry.json +15 -0
- package/src/config/support-tiers/subsystems.json +502 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
- package/src/config/transforms/capability-transform-resolver.js +44 -9
- package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
- package/src/config/transforms/execution-graph-transforms.js +1220 -8
- package/src/converter/conversion-plan.js +20 -8
- package/src/converter/core.d.ts +48 -1
- package/src/converter/core.js +249 -62
- package/src/converter/parsers/gguf.d.ts +10 -1
- package/src/converter/rope-config.d.ts +1 -0
- package/src/converter/rope-config.js +75 -5
- package/src/converter/shard-packer.js +2 -2
- package/src/debug/tensor.js +13 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
- package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
- package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
- package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
- package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
- package/src/{bridge → experimental/bridge}/protocol.js +1 -1
- package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
- package/src/{browser → experimental/browser}/browser-converter.js +12 -12
- package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
- package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
- package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
- package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
- package/src/{browser → experimental/browser}/quantization.js +4 -4
- package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
- package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
- package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
- package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
- package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
- package/src/experimental/diffusion/index.d.ts +8 -0
- package/src/experimental/diffusion/index.js +8 -0
- package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
- package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
- package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
- package/src/experimental/energy/index.d.ts +2 -0
- package/src/experimental/energy/index.js +2 -0
- package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
- package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
- package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
- package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
- package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
- package/src/experimental/orchestration/index.d.ts +12 -0
- package/src/experimental/orchestration/index.js +12 -0
- package/src/{training → experimental/training}/README.md +1 -1
- package/src/{training → experimental/training}/artifacts.d.ts +3 -3
- package/src/{training → experimental/training}/artifacts.js +3 -3
- package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
- package/src/{training → experimental/training}/attention-backward.js +3 -3
- package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
- package/src/{training → experimental/training}/attention-forward.js +1 -1
- package/src/{training → experimental/training}/autograd.d.ts +2 -2
- package/src/{training → experimental/training}/autograd.js +7 -7
- package/src/{training → experimental/training}/backward-kernels.js +1 -1
- package/src/{training → experimental/training}/checkpoint.js +1 -1
- package/src/{training → experimental/training}/clip.d.ts +2 -2
- package/src/{training → experimental/training}/clip.js +3 -3
- package/src/{training → experimental/training}/export.d.ts +2 -2
- package/src/{training → experimental/training}/export.js +2 -2
- package/src/{training → experimental/training}/lora-pipeline.js +4 -4
- package/src/{training → experimental/training}/lora.d.ts +1 -1
- package/src/{training → experimental/training}/lora.js +4 -4
- package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
- package/src/{training → experimental/training}/loss-scaling.js +3 -3
- package/src/{training → experimental/training}/loss.d.ts +2 -2
- package/src/{training → experimental/training}/loss.js +1 -1
- package/src/{training → experimental/training}/operator-artifacts.js +1 -1
- package/src/{training → experimental/training}/optimizer.d.ts +2 -2
- package/src/{training → experimental/training}/optimizer.js +3 -3
- package/src/{training → experimental/training}/runner.d.ts +2 -2
- package/src/{training → experimental/training}/runner.js +5 -5
- package/src/{training → experimental/training}/suite.js +16 -16
- package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
- package/src/{training → experimental/training}/tensor-factory.js +2 -2
- package/src/{training → experimental/training}/trainer.d.ts +3 -3
- package/src/{training → experimental/training}/trainer.js +3 -3
- package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
- package/src/{training → experimental/training}/ul_dataset.js +1 -1
- package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
- package/src/{training → experimental/training}/workloads.js +4 -4
- package/src/formats/index.d.ts +24 -0
- package/src/formats/index.js +6 -0
- package/src/formats/litert/index.d.ts +1 -0
- package/src/formats/litert/index.js +1 -0
- package/src/formats/litert/types.d.ts +87 -0
- package/src/formats/litert/types.js +568 -0
- package/src/formats/rdrr/classification.js +16 -8
- package/src/formats/rdrr/parsing.d.ts +3 -2
- package/src/formats/rdrr/types.d.ts +33 -0
- package/src/formats/rdrr/validation.js +16 -5
- package/src/formats/tflite/index.d.ts +1 -0
- package/src/formats/tflite/index.js +1 -0
- package/src/formats/tflite/types.d.ts +84 -0
- package/src/formats/tflite/types.js +633 -0
- package/src/generation/index.d.ts +16 -14
- package/src/generation/index.js +1 -32
- package/src/gpu/command-recorder.d.ts +17 -1
- package/src/gpu/command-recorder.js +47 -10
- package/src/gpu/device.d.ts +3 -1
- package/src/gpu/device.js +17 -11
- package/src/gpu/kernel-selector.js +3 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +359 -14
- package/src/gpu/kernels/attention.wgsl +20 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
- package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
- package/src/gpu/kernels/attention_streaming.wgsl +22 -3
- package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
- package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
- package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
- package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
- package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
- package/src/gpu/kernels/depthwise_conv1d.js +51 -0
- package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
- package/src/gpu/kernels/feature-check.js +0 -1
- package/src/gpu/kernels/fused_ffn.js +67 -22
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
- package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
- package/src/gpu/kernels/gather.d.ts +4 -0
- package/src/gpu/kernels/gather.js +13 -2
- package/src/gpu/kernels/gather.wgsl +5 -3
- package/src/gpu/kernels/gather_f16.wgsl +5 -3
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/gpu/kernels/kernel-configs.js +1 -0
- package/src/gpu/kernels/kv-quantize.js +87 -55
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
- package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
- package/src/gpu/kernels/linear-attention-core.js +145 -56
- package/src/gpu/kernels/matmul-dispatch.js +3 -0
- package/src/gpu/kernels/matmul-selection.js +31 -42
- package/src/gpu/kernels/matmul.d.ts +3 -0
- package/src/gpu/kernels/matmul.js +72 -34
- package/src/gpu/kernels/residual.d.ts +2 -0
- package/src/gpu/kernels/residual.js +43 -4
- package/src/gpu/kernels/rope.d.ts +2 -0
- package/src/gpu/kernels/rope.js +19 -6
- package/src/gpu/kernels/rope.wgsl +24 -12
- package/src/gpu/kernels/rope_f16.wgsl +97 -38
- package/src/gpu/kernels/scale.d.ts +5 -0
- package/src/gpu/kernels/scale.js +6 -1
- package/src/gpu/kernels/scale.wgsl +3 -1
- package/src/gpu/kernels/scale_f16.wgsl +32 -0
- package/src/gpu/kernels/turboquant-codebook.js +564 -0
- package/src/gpu/kernels/uniform-utils.js +26 -0
- package/src/gpu/kernels/utils.js +6 -1
- package/src/gpu/submit-probe.d.ts +12 -0
- package/src/gpu/submit-probe.js +77 -0
- package/src/gpu/weight-buffer.d.ts +26 -3
- package/src/gpu/weight-buffer.js +62 -2
- package/src/index-browser.d.ts +1 -91
- package/src/index-browser.js +0 -66
- package/src/index.d.ts +1 -102
- package/src/index.js +0 -74
- package/src/inference/browser-harness-model-helpers.d.ts +3 -0
- package/src/inference/browser-harness-model-helpers.js +134 -11
- package/src/inference/browser-harness-runtime-helpers.js +1 -9
- package/src/inference/browser-harness-text-helpers.js +441 -70
- package/src/inference/browser-harness.js +61 -4
- package/src/inference/decode-ring.d.ts +2 -0
- package/src/inference/decode-ring.js +20 -0
- package/src/inference/kv-cache/index.d.ts +3 -0
- package/src/inference/kv-cache/index.js +2 -0
- package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
- package/src/inference/kv-cache/mixed-geometry.js +422 -0
- package/src/inference/kv-cache/quantized.d.ts +56 -0
- package/src/inference/kv-cache/quantized.js +383 -0
- package/src/inference/kv-cache/tiered.d.ts +20 -3
- package/src/inference/kv-cache/tiered.js +203 -65
- package/src/inference/kv-cache/types.d.ts +49 -6
- package/src/inference/kv-cache.d.ts +3 -0
- package/src/inference/kv-cache.js +2 -0
- package/src/inference/multi-model-network.js +2 -1
- package/src/inference/pipelines/audio/gemma4.js +655 -0
- package/src/inference/pipelines/audio/index.js +39 -0
- package/src/inference/pipelines/audio/mel.js +171 -0
- package/src/inference/pipelines/context.js +9 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
- package/src/inference/pipelines/shared/clipped-linear.js +46 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
- package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
- package/src/inference/pipelines/text/attention/output-projection.js +3 -3
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
- package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
- package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
- package/src/inference/pipelines/text/attention/projections.js +182 -15
- package/src/inference/pipelines/text/attention/quant-options.js +54 -0
- package/src/inference/pipelines/text/attention/record.js +257 -69
- package/src/inference/pipelines/text/attention/run.js +306 -94
- package/src/inference/pipelines/text/attention/types.d.ts +11 -0
- package/src/inference/pipelines/text/chat-format.d.ts +8 -1
- package/src/inference/pipelines/text/chat-format.js +89 -5
- package/src/inference/pipelines/text/config.d.ts +78 -26
- package/src/inference/pipelines/text/config.js +722 -28
- package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
- package/src/inference/pipelines/text/dtype-contract.js +38 -0
- package/src/inference/pipelines/text/embed.d.ts +4 -0
- package/src/inference/pipelines/text/embed.js +165 -172
- package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
- package/src/inference/pipelines/text/execution-plan.js +112 -20
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
- package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
- package/src/inference/pipelines/text/execution-v1.js +411 -33
- package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
- package/src/inference/pipelines/text/ffn/dense.js +575 -63
- package/src/inference/pipelines/text/ffn/moe.js +1 -0
- package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
- package/src/inference/pipelines/text/ffn/standard.js +28 -4
- package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
- package/src/inference/pipelines/text/finiteness-policy.js +14 -11
- package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
- package/src/inference/pipelines/text/generator-helpers.js +39 -4
- package/src/inference/pipelines/text/generator-runtime.js +45 -8
- package/src/inference/pipelines/text/generator-steps.js +407 -75
- package/src/inference/pipelines/text/generator.d.ts +6 -0
- package/src/inference/pipelines/text/generator.js +1179 -305
- package/src/inference/pipelines/text/init.d.ts +39 -7
- package/src/inference/pipelines/text/init.js +525 -110
- package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
- package/src/inference/pipelines/text/layer-plan.js +2 -0
- package/src/inference/pipelines/text/layer.js +402 -19
- package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
- package/src/inference/pipelines/text/linear-attention.js +112 -41
- package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
- package/src/inference/pipelines/text/logits/gpu.js +334 -15
- package/src/inference/pipelines/text/logits/index.d.ts +11 -0
- package/src/inference/pipelines/text/logits/index.js +227 -29
- package/src/inference/pipelines/text/logits/types.d.ts +3 -0
- package/src/inference/pipelines/text/logits/utils.js +3 -1
- package/src/inference/pipelines/text/lora-apply.js +3 -3
- package/src/inference/pipelines/text/model-load.d.ts +2 -1
- package/src/inference/pipelines/text/model-load.js +205 -178
- package/src/inference/pipelines/text/moe-gpu.js +8 -0
- package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
- package/src/inference/pipelines/text/ops.d.ts +16 -2
- package/src/inference/pipelines/text/ops.js +11 -2
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
- package/src/inference/pipelines/text/probes.js +6 -0
- package/src/inference/pipelines/text/sampling-config.js +9 -1
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.d.ts +12 -3
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/types.d.ts +79 -7
- package/src/inference/pipelines/text/weights.d.ts +2 -2
- package/src/inference/pipelines/text/weights.js +4 -0
- package/src/inference/pipelines/text.d.ts +6 -0
- package/src/inference/pipelines/text.js +860 -11
- package/src/inference/pipelines/video/frame-sampler.js +36 -0
- package/src/inference/pipelines/video/index.js +87 -0
- package/src/inference/pipelines/vision/encoder.js +27 -6
- package/src/inference/pipelines/vision/gemma4.js +731 -0
- package/src/inference/pipelines/vision/image-preprocess.js +22 -7
- package/src/inference/pipelines/vision/index.js +35 -10
- package/src/inference/pipelines/vision/ops.js +31 -26
- package/src/inference/pipelines/vision/patch-embed.js +22 -10
- package/src/inference/runtime-model.d.ts +89 -0
- package/src/inference/runtime-model.js +152 -0
- package/src/inference/test-harness.d.ts +8 -1
- package/src/inference/test-harness.js +54 -20
- package/src/inference/tokenizer.d.ts +5 -0
- package/src/inference/tokenizer.js +43 -0
- package/src/inference/tokenizers/base.d.ts +5 -0
- package/src/inference/tokenizers/base.js +5 -0
- package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
- package/src/inference/tokenizers/behavior-flags.js +100 -0
- package/src/inference/tokenizers/bundled.d.ts +4 -0
- package/src/inference/tokenizers/bundled.js +110 -0
- package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
- package/src/inference/tokenizers/sentencepiece.js +16 -0
- package/src/inference/tokenizers/types.d.ts +2 -0
- package/src/loader/doppler-loader.d.ts +10 -3
- package/src/loader/doppler-loader.js +103 -5
- package/src/loader/embedding-loader.d.ts +5 -1
- package/src/loader/embedding-loader.js +57 -6
- package/src/loader/final-weights-loader.js +44 -5
- package/src/loader/layer-loader.js +210 -8
- package/src/loader/loader-state.js +2 -2
- package/src/loader/loader-types.d.ts +17 -1
- package/src/loader/memory-monitor.d.ts +8 -0
- package/src/loader/memory-monitor.js +21 -0
- package/src/loader/multi-model-loader.d.ts +9 -1
- package/src/loader/multi-model-loader.js +9 -1
- package/src/loader/per-layer-input-loader.d.ts +30 -0
- package/src/loader/per-layer-input-loader.js +478 -0
- package/src/loader/shard-cache.d.ts +4 -3
- package/src/loader/shard-resolver.js +2 -0
- package/src/loader/tensors/source-transform.d.ts +50 -0
- package/src/loader/tensors/source-transform.js +776 -0
- package/src/loader/tensors/tensor-loader.d.ts +12 -0
- package/src/loader/tensors/tensor-loader.js +208 -123
- package/src/loader/tensors/tensor-reader.d.ts +8 -0
- package/src/loader/tensors/tensor-reader.js +283 -15
- package/src/loaders/index.d.ts +4 -0
- package/src/loaders/index.js +4 -0
- package/src/memory/buffer-pool.d.ts +4 -0
- package/src/memory/buffer-pool.js +36 -0
- package/src/rules/execution-rules-contract-check.d.ts +4 -0
- package/src/rules/execution-rules-contract-check.js +150 -24
- package/src/rules/inference/attention.rules.json +15 -0
- package/src/rules/inference/capability-transforms.rules.json +30 -0
- package/src/rules/inference/execution.rules.json +54 -2
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/attention.rules.json +7 -0
- package/src/rules/kernels/fused-ffn.rules.json +4 -2
- package/src/rules/kernels/kv_quantize.rules.json +2 -0
- package/src/rules/kernels/matmul.rules.json +14 -0
- package/src/rules/kernels/scale.rules.json +2 -0
- package/src/rules/loader/tensor-loader.rules.json +1 -0
- package/src/storage/artifact-storage-context.d.ts +53 -0
- package/src/storage/artifact-storage-context.js +560 -0
- package/src/storage/downloader.js +7 -1
- package/src/storage/index.d.ts +18 -0
- package/src/storage/index.js +13 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +35 -2
- package/src/storage/registry.d.ts +1 -0
- package/src/storage/registry.js +26 -4
- package/src/storage/shard-manager.d.ts +4 -0
- package/src/storage/shard-manager.js +25 -6
- package/src/storage/source-artifact-store.d.ts +9 -0
- package/src/storage/source-artifact-store.js +134 -0
- package/src/tooling/browser-command-runner.js +12 -5
- package/src/tooling/command-api-family-normalizers.js +144 -0
- package/src/tooling/command-api-helpers.js +1 -0
- package/src/tooling/command-api.d.ts +32 -9
- package/src/tooling/command-api.js +0 -25
- package/src/tooling/command-runner-shared.js +1 -2
- package/src/tooling/command-runner.html +5 -2
- package/src/tooling/diagnose-runner.js +1 -1
- package/src/tooling/kernel-path-builder/index.js +48 -32
- package/src/tooling/lean-execution-contract.d.ts +1 -1
- package/src/tooling/lean-execution-contract.js +2 -149
- package/src/tooling/litert-package-runtime.d.ts +46 -0
- package/src/tooling/litert-package-runtime.js +950 -0
- package/src/tooling/node-browser-command-runner.d.ts +13 -0
- package/src/tooling/node-browser-command-runner.js +163 -3
- package/src/tooling/node-command-runner.js +14 -5
- package/src/tooling/node-converter.js +226 -63
- package/src/tooling/node-source-runtime.d.ts +8 -3
- package/src/tooling/node-source-runtime.js +561 -236
- package/src/tooling/node-webgpu.js +7 -30
- package/src/tooling/opfs-cache.d.ts +18 -1
- package/src/tooling/opfs-cache.js +126 -9
- package/src/tooling/runtime-input-composition.d.ts +0 -1
- package/src/tooling/runtime-input-composition.js +0 -5
- package/src/tooling/source-artifact-adapter.d.ts +134 -0
- package/src/tooling/source-artifact-adapter.js +366 -0
- package/src/tooling/source-package-profiles.d.ts +33 -0
- package/src/tooling/source-package-profiles.js +57 -0
- package/src/tooling/source-runtime-bundle.d.ts +46 -6
- package/src/tooling/source-runtime-bundle.js +371 -60
- package/src/tooling/source-runtime-converter-config.d.ts +24 -0
- package/src/tooling/source-runtime-converter-config.js +297 -0
- package/src/tooling-experimental-exports.browser.js +2 -0
- package/src/tooling-experimental-exports.d.ts +7 -0
- package/src/tooling-experimental-exports.js +7 -0
- package/src/tooling-experimental-exports.shared.d.ts +26 -0
- package/src/tooling-experimental-exports.shared.js +29 -0
- package/src/tooling-exports.shared.d.ts +0 -23
- package/src/tooling-exports.shared.js +6 -28
- package/src/version.js +1 -1
- package/src/client/doppler-provider/generation.d.ts +0 -25
- package/src/client/doppler-provider/generation.js +0 -126
- package/src/client/doppler-provider/index.d.ts +0 -2
- package/src/client/doppler-provider/index.js +0 -3
- package/src/client/doppler-provider/provider.d.ts +0 -5
- package/src/client/doppler-provider/provider.js +0 -102
- package/src/client/doppler-provider/source-runtime.js +0 -677
- package/src/client/doppler-provider.d.ts +0 -46
- package/src/client/doppler-provider.js +0 -36
- package/src/config/schema/inference-defaults.schema.d.ts +0 -276
- package/src/config/schema/inference-defaults.schema.js +0 -189
- package/src/diffusion/index.d.ts +0 -8
- package/src/diffusion/index.js +0 -8
- package/src/energy/index.d.ts +0 -2
- package/src/energy/index.js +0 -2
- package/src/index-internal.d.ts +0 -2
- package/src/index-internal.js +0 -2
- /package/src/client/{doppler-provider → runtime}/types.js +0 -0
- /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
- /package/src/{adapters → experimental/adapters}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
- /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
- /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
- /package/src/{bridge → experimental/bridge}/index.js +0 -0
- /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
- /package/src/{browser → experimental/browser}/file-picker.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
- /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
- /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
- /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
- /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
- /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
- /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
- /package/src/{training → experimental/training}/dataloader.js +0 -0
- /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/index.js +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
- /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
- /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/eval.js +0 -0
- /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/index.js +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
- /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
- /package/src/{training → experimental/training}/index.d.ts +0 -0
- /package/src/{training → experimental/training}/index.js +0 -0
- /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/base.js +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
- /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/index.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
- /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
- /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-command.js +0 -0
- /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-eval.js +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
- /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
- /package/src/{training → experimental/training}/suite.d.ts +0 -0
- /package/src/{training → experimental/training}/ul_schedule.js +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
- /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
- /package/src/{training → experimental/training}/workloads.d.ts +0 -0
|
@@ -31,6 +31,21 @@ const SUBGROUP_REQUIRING_FILES = new Set([
|
|
|
31
31
|
'attention_decode_online_f16.wgsl',
|
|
32
32
|
]);
|
|
33
33
|
|
|
34
|
+
const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
35
|
+
['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
36
|
+
['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
37
|
+
['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
38
|
+
['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
39
|
+
['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
40
|
+
['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
41
|
+
['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
42
|
+
['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
43
|
+
['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
44
|
+
['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
45
|
+
['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
46
|
+
['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
47
|
+
]);
|
|
48
|
+
|
|
34
49
|
/**
|
|
35
50
|
* Check whether a kernel entry requires subgroup support.
|
|
36
51
|
* @param {{ kernel: string }} kernelEntry
|
|
@@ -41,6 +56,11 @@ function isSubgroupKernel(kernelEntry) {
|
|
|
41
56
|
return kernelEntry.kernel.includes('subgroup') || SUBGROUP_REQUIRING_FILES.has(kernelEntry.kernel);
|
|
42
57
|
}
|
|
43
58
|
|
|
59
|
+
function requiresNoSubgroupFallback(kernelEntry) {
|
|
60
|
+
if (typeof kernelEntry?.kernel !== 'string') return false;
|
|
61
|
+
return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
|
|
62
|
+
}
|
|
63
|
+
|
|
44
64
|
/**
|
|
45
65
|
* Find all kernel keys in the graph whose `kernel` file matches the given filename.
|
|
46
66
|
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
@@ -82,9 +102,39 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
|
82
102
|
} else if (constants !== undefined) {
|
|
83
103
|
derived.constants = { ...constants };
|
|
84
104
|
}
|
|
105
|
+
const precision = deriveKernelPrecision(base, newFile);
|
|
106
|
+
if (precision) {
|
|
107
|
+
derived.precision = precision;
|
|
108
|
+
} else {
|
|
109
|
+
delete derived.precision;
|
|
110
|
+
}
|
|
85
111
|
return derived;
|
|
86
112
|
}
|
|
87
113
|
|
|
114
|
+
function deriveKernelPrecision(base, newFile) {
|
|
115
|
+
const precision = base.precision ? { ...base.precision } : {};
|
|
116
|
+
const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
|
|
117
|
+
if (precisionPatch) {
|
|
118
|
+
Object.assign(precision, precisionPatch);
|
|
119
|
+
}
|
|
120
|
+
if (!String(newFile).startsWith('attention')) {
|
|
121
|
+
return Object.keys(precision).length > 0 ? precision : null;
|
|
122
|
+
}
|
|
123
|
+
if (newFile.includes('_f16kv')) {
|
|
124
|
+
precision.activationDtype = 'f32';
|
|
125
|
+
precision.kvDtype = 'f16';
|
|
126
|
+
return precision;
|
|
127
|
+
}
|
|
128
|
+
if (newFile.includes('_f16')) {
|
|
129
|
+
precision.activationDtype = 'f16';
|
|
130
|
+
precision.kvDtype = 'f16';
|
|
131
|
+
return precision;
|
|
132
|
+
}
|
|
133
|
+
precision.activationDtype = 'f32';
|
|
134
|
+
precision.kvDtype = 'f32';
|
|
135
|
+
return precision;
|
|
136
|
+
}
|
|
137
|
+
|
|
88
138
|
/**
|
|
89
139
|
* Derive a non-colliding kernel key name.
|
|
90
140
|
* @param {object} kernels - existing kernels dict
|
|
@@ -136,6 +186,292 @@ function stepUsesFile(graph, step, filename) {
|
|
|
136
186
|
return entry != null && entry.kernel === filename;
|
|
137
187
|
}
|
|
138
188
|
|
|
189
|
+
/**
|
|
190
|
+
* Find the first kernel key used by matching ops in a phase whose shader file
|
|
191
|
+
* satisfies the provided predicate.
|
|
192
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
193
|
+
* @param {Array<Array>} steps
|
|
194
|
+
* @param {ReadonlySet<string>} ops
|
|
195
|
+
* @param {(entry: { kernel: string, entry: string }) => boolean} predicate
|
|
196
|
+
* @returns {string | null}
|
|
197
|
+
*/
|
|
198
|
+
function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
199
|
+
for (const step of steps || []) {
|
|
200
|
+
if (!ops.has(step[0])) {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
const entry = graph.kernels[step[1]];
|
|
204
|
+
if (entry && predicate(entry)) {
|
|
205
|
+
return step[1];
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Find an existing kernel key by shader file and entry point.
|
|
213
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
214
|
+
* @param {string} filename
|
|
215
|
+
* @param {string} entryPoint
|
|
216
|
+
* @returns {string | null}
|
|
217
|
+
*/
|
|
218
|
+
function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
|
|
219
|
+
for (const [key, entry] of Object.entries(graph.kernels)) {
|
|
220
|
+
if (entry.kernel === filename && entry.entry === entryPoint) {
|
|
221
|
+
return key;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function normalizeLayerType(layerType) {
|
|
228
|
+
return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function isLinearAttentionLayerType(layerType) {
|
|
232
|
+
const normalized = normalizeLayerType(layerType);
|
|
233
|
+
return normalized === 'linear_attention'
|
|
234
|
+
|| normalized === 'linear'
|
|
235
|
+
|| normalized === 'gated_delta'
|
|
236
|
+
|| normalized === 'gated_delta_net';
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function isFullAttentionLayerType(layerType) {
|
|
240
|
+
const normalized = normalizeLayerType(layerType);
|
|
241
|
+
return normalized === 'full_attention'
|
|
242
|
+
|| normalized === 'full'
|
|
243
|
+
|| normalized === 'global'
|
|
244
|
+
|| normalized === 'standard';
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey) {
|
|
248
|
+
const groupedEntries = [];
|
|
249
|
+
if (!Array.isArray(baseStep) || baseStep.length < 2) {
|
|
250
|
+
return groupedEntries;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const totalLayers = targetLayers.allLayers;
|
|
254
|
+
const targeted = targetLayers.matchingLayers;
|
|
255
|
+
const remaining = totalLayers.filter((layerIdx) => !targeted.includes(layerIdx));
|
|
256
|
+
|
|
257
|
+
if (remaining.length > 0) {
|
|
258
|
+
groupedEntries.push({
|
|
259
|
+
layers: remaining,
|
|
260
|
+
steps: [baseStep],
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
if (targeted.length > 0) {
|
|
264
|
+
const replacement = [...baseStep];
|
|
265
|
+
replacement[1] = replacementKernelKey;
|
|
266
|
+
groupedEntries.push({
|
|
267
|
+
layers: targeted,
|
|
268
|
+
steps: [replacement],
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return groupedEntries;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
|
|
276
|
+
if (!Array.isArray(steps) || steps.length === 0 || !replacementKernelKey) {
|
|
277
|
+
return { steps, changed: false };
|
|
278
|
+
}
|
|
279
|
+
let changed = false;
|
|
280
|
+
const nextSteps = steps.map((step) => {
|
|
281
|
+
if (!Array.isArray(step) || step[0] !== op) {
|
|
282
|
+
return step;
|
|
283
|
+
}
|
|
284
|
+
if (step[1] === replacementKernelKey) {
|
|
285
|
+
return step;
|
|
286
|
+
}
|
|
287
|
+
const replacement = [...step];
|
|
288
|
+
replacement[1] = replacementKernelKey;
|
|
289
|
+
changed = true;
|
|
290
|
+
return replacement;
|
|
291
|
+
});
|
|
292
|
+
return { steps: nextSteps, changed };
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function deriveKernelEntryWithPrecision(base, precision) {
|
|
296
|
+
return {
|
|
297
|
+
...base,
|
|
298
|
+
precision: {
|
|
299
|
+
...(base.precision ?? {}),
|
|
300
|
+
...precision,
|
|
301
|
+
},
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const ATTENTION_F16KV_TO_F16_MAP = new Map([
|
|
306
|
+
['attention_decode_online_f16kv.wgsl', 'attention_decode_online_f16.wgsl'],
|
|
307
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
|
|
308
|
+
['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
|
|
309
|
+
['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
|
|
310
|
+
]);
|
|
311
|
+
|
|
312
|
+
function deriveF16AttentionKernelEntry(base) {
|
|
313
|
+
if (typeof base?.kernel !== 'string') {
|
|
314
|
+
return null;
|
|
315
|
+
}
|
|
316
|
+
const replacement = ATTENTION_F16KV_TO_F16_MAP.get(base.kernel);
|
|
317
|
+
if (!replacement) {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
return deriveKernelEntryWithPrecision(
|
|
321
|
+
deriveKernelEntry(base, replacement, base.entry),
|
|
322
|
+
{ activationDtype: 'f16', kvDtype: 'f16' }
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function deriveLinearDecodeF16KernelEntry(base) {
|
|
327
|
+
const precision = {
|
|
328
|
+
inputDtype: 'f16',
|
|
329
|
+
outputDtype: 'f16',
|
|
330
|
+
};
|
|
331
|
+
if (base.kernel === 'fused_matmul_q4.wgsl' && base.entry === 'main_multicol') {
|
|
332
|
+
return {
|
|
333
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a'),
|
|
334
|
+
precision: {
|
|
335
|
+
...(base.precision ?? {}),
|
|
336
|
+
...precision,
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
if (
|
|
341
|
+
(base.kernel === 'fused_matmul_q4_multicol_f16.wgsl' && base.entry === 'main_multicol_f16')
|
|
342
|
+
|| (base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl' && base.entry === 'main_multicol_f16a')
|
|
343
|
+
) {
|
|
344
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
345
|
+
}
|
|
346
|
+
return null;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function deriveLmHeadDecodeF16KernelEntry(base) {
|
|
350
|
+
const precision = {
|
|
351
|
+
inputDtype: 'f16',
|
|
352
|
+
outputDtype: 'f16',
|
|
353
|
+
};
|
|
354
|
+
if (base.kernel === 'matmul_gemv_subgroup.wgsl' && base.entry === 'main_multicol') {
|
|
355
|
+
return {
|
|
356
|
+
...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', 'main_multicol'),
|
|
357
|
+
precision: {
|
|
358
|
+
...(base.precision ?? {}),
|
|
359
|
+
...precision,
|
|
360
|
+
},
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl' && base.entry === 'main_multicol') {
|
|
364
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
365
|
+
}
|
|
366
|
+
return null;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function deriveQ4DecodeF16KernelEntry(base) {
|
|
370
|
+
if (typeof base?.kernel !== 'string') {
|
|
371
|
+
return null;
|
|
372
|
+
}
|
|
373
|
+
const precision = {
|
|
374
|
+
inputDtype: 'f16',
|
|
375
|
+
outputDtype: 'f16',
|
|
376
|
+
};
|
|
377
|
+
if (base.kernel === 'fused_matmul_q4.wgsl') {
|
|
378
|
+
return {
|
|
379
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a', null),
|
|
380
|
+
precision: {
|
|
381
|
+
...(base.precision ?? {}),
|
|
382
|
+
...precision,
|
|
383
|
+
},
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
if (
|
|
387
|
+
base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
|
|
388
|
+
|| base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
|
|
389
|
+
) {
|
|
390
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
391
|
+
}
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function deriveQ4PrefillF16KernelEntry(base) {
|
|
396
|
+
if (typeof base?.kernel !== 'string') {
|
|
397
|
+
return null;
|
|
398
|
+
}
|
|
399
|
+
const precision = {
|
|
400
|
+
inputDtype: 'f16',
|
|
401
|
+
outputDtype: 'f16',
|
|
402
|
+
};
|
|
403
|
+
if (base.kernel.startsWith('fused_matmul_q4_batched')) {
|
|
404
|
+
return {
|
|
405
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
|
|
406
|
+
precision: {
|
|
407
|
+
...(base.precision ?? {}),
|
|
408
|
+
...precision,
|
|
409
|
+
},
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
416
|
+
if (typeof base?.kernel !== 'string') {
|
|
417
|
+
return null;
|
|
418
|
+
}
|
|
419
|
+
const precision = {
|
|
420
|
+
inputDtype: 'f32',
|
|
421
|
+
outputDtype: 'f32',
|
|
422
|
+
};
|
|
423
|
+
if (
|
|
424
|
+
base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
|
|
425
|
+
|| base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
|
|
426
|
+
) {
|
|
427
|
+
return deriveKernelEntryWithPrecision(
|
|
428
|
+
deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main_multicol', null),
|
|
429
|
+
precision
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
if (base.kernel === 'fused_matmul_q4_f16a.wgsl') {
|
|
433
|
+
return deriveKernelEntryWithPrecision(
|
|
434
|
+
deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main', null),
|
|
435
|
+
precision
|
|
436
|
+
);
|
|
437
|
+
}
|
|
438
|
+
return null;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function deriveQ4PrefillF32ActivationKernelEntry(base) {
|
|
442
|
+
if (typeof base?.kernel !== 'string') {
|
|
443
|
+
return null;
|
|
444
|
+
}
|
|
445
|
+
if (
|
|
446
|
+
base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
|
|
447
|
+
&& base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
|
|
448
|
+
) {
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
return deriveKernelEntryWithPrecision(
|
|
452
|
+
deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
|
|
453
|
+
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function replacePhaseStepEntries(steps, op, replacementEntries) {
|
|
458
|
+
if (!Array.isArray(steps) || steps.length === 0 || !Array.isArray(replacementEntries) || replacementEntries.length === 0) {
|
|
459
|
+
return { steps, changed: false };
|
|
460
|
+
}
|
|
461
|
+
const stepIndex = steps.findIndex((entry) => Array.isArray(entry) && entry[0] === op);
|
|
462
|
+
if (stepIndex === -1) {
|
|
463
|
+
return { steps, changed: false };
|
|
464
|
+
}
|
|
465
|
+
return {
|
|
466
|
+
steps: [
|
|
467
|
+
...steps.slice(0, stepIndex),
|
|
468
|
+
...replacementEntries,
|
|
469
|
+
...steps.slice(stepIndex + 1),
|
|
470
|
+
],
|
|
471
|
+
changed: true,
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
|
|
139
475
|
// =============================================================================
|
|
140
476
|
// Transform: removeSubgroups
|
|
141
477
|
// =============================================================================
|
|
@@ -151,8 +487,8 @@ function stepUsesFile(graph, step, filename) {
|
|
|
151
487
|
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
152
488
|
*/
|
|
153
489
|
export function removeSubgroups(graph, ctx) {
|
|
154
|
-
const
|
|
155
|
-
if (!
|
|
490
|
+
const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
|
|
491
|
+
if (!hasAnyFallbackKernel) {
|
|
156
492
|
return null;
|
|
157
493
|
}
|
|
158
494
|
|
|
@@ -160,15 +496,16 @@ export function removeSubgroups(graph, ctx) {
|
|
|
160
496
|
const keyMap = new Map();
|
|
161
497
|
const isF16Activation = ctx.activationDtype === 'f16';
|
|
162
498
|
|
|
163
|
-
// Build replacement kernel entries for each subgroup kernel
|
|
164
|
-
// found in decode and postLayer steps.
|
|
499
|
+
// Build replacement kernel entries for each subgroup or fused-Q4K kernel
|
|
500
|
+
// reference found in decode, prefill, and postLayer steps.
|
|
165
501
|
const decodeKeys = new Set((result.decode || []).map((s) => s[1]));
|
|
502
|
+
const prefillKeys = new Set((result.prefill || []).map((s) => s[1]));
|
|
166
503
|
const postLayerKeys = new Set((result.postLayer || []).map((s) => s[1]));
|
|
167
|
-
const relevantKeys = new Set([...decodeKeys, ...postLayerKeys]);
|
|
504
|
+
const relevantKeys = new Set([...decodeKeys, ...prefillKeys, ...postLayerKeys]);
|
|
168
505
|
|
|
169
506
|
for (const key of relevantKeys) {
|
|
170
507
|
const entry = result.kernels[key];
|
|
171
|
-
if (!entry || !
|
|
508
|
+
if (!entry || !requiresNoSubgroupFallback(entry)) {
|
|
172
509
|
continue;
|
|
173
510
|
}
|
|
174
511
|
|
|
@@ -205,6 +542,9 @@ export function removeSubgroups(graph, ctx) {
|
|
|
205
542
|
} else if (entry.kernel === 'attention_decode_online_f16.wgsl') {
|
|
206
543
|
newFile = 'attention_decode_chunked_f16.wgsl';
|
|
207
544
|
newEntry = entry.entry;
|
|
545
|
+
} else if (entry.kernel.startsWith('fused_matmul_q4')) {
|
|
546
|
+
newFile = isF16Activation ? 'matmul_f16_tiled.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
|
|
547
|
+
newConstants = null;
|
|
208
548
|
} else {
|
|
209
549
|
// Unknown subgroup kernel — skip
|
|
210
550
|
continue;
|
|
@@ -219,8 +559,9 @@ export function removeSubgroups(graph, ctx) {
|
|
|
219
559
|
return null;
|
|
220
560
|
}
|
|
221
561
|
|
|
222
|
-
// Remap decode and postLayer steps; leave
|
|
562
|
+
// Remap decode, prefill, and postLayer steps; leave preLayer untouched
|
|
223
563
|
result.decode = remapStepKeys(result.decode || [], keyMap);
|
|
564
|
+
result.prefill = remapStepKeys(result.prefill || [], keyMap);
|
|
224
565
|
result.postLayer = remapStepKeys(result.postLayer || [], keyMap);
|
|
225
566
|
|
|
226
567
|
return result;
|
|
@@ -252,6 +593,55 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
252
593
|
['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
253
594
|
]);
|
|
254
595
|
|
|
596
|
+
/**
|
|
597
|
+
* Activation-only narrowing: f32-activation shaders that still consume f16
|
|
598
|
+
* weights/KV are rewritten onto the matching f16-activation lane.
|
|
599
|
+
*
|
|
600
|
+
* This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
|
|
601
|
+
* runtime session explicitly requests f16 activations for an execution-v1
|
|
602
|
+
* graph that was authored with conservative f32 activation defaults.
|
|
603
|
+
* @type {ReadonlyMap<string, string>}
|
|
604
|
+
*/
|
|
605
|
+
const F32_TO_F16_ACTIVATION_MAP = new Map(
|
|
606
|
+
Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
|
|
607
|
+
);
|
|
608
|
+
|
|
609
|
+
function hasExplicitF32ActivationContract(entry) {
|
|
610
|
+
const precision = entry?.precision;
|
|
611
|
+
if (!precision || typeof precision !== 'object') {
|
|
612
|
+
return false;
|
|
613
|
+
}
|
|
614
|
+
return precision.activationDtype === 'f32'
|
|
615
|
+
|| precision.inputDtype === 'f32'
|
|
616
|
+
|| precision.outputDtype === 'f32';
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Correctness fallback: preserve f16 weights where possible, but widen both
|
|
621
|
+
* activations and KV-cache interactions onto the stable f32 execution lane.
|
|
622
|
+
* Used for alternate-plan recovery after finiteness failure.
|
|
623
|
+
* @type {ReadonlyMap<string, string>}
|
|
624
|
+
*/
|
|
625
|
+
const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
626
|
+
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
627
|
+
['rope_f16.wgsl', 'rope.wgsl'],
|
|
628
|
+
['residual_f16.wgsl', 'residual.wgsl'],
|
|
629
|
+
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
630
|
+
['sample_f16.wgsl', 'sample.wgsl'],
|
|
631
|
+
['gather_f16.wgsl', 'gather.wgsl'],
|
|
632
|
+
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
633
|
+
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
634
|
+
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
635
|
+
['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
|
|
636
|
+
['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
|
|
637
|
+
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
638
|
+
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
639
|
+
['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
|
|
640
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
|
|
641
|
+
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
642
|
+
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
643
|
+
]);
|
|
644
|
+
|
|
255
645
|
/**
|
|
256
646
|
* Full f32 widening: every shader that uses `enable f16;` is replaced with a
|
|
257
647
|
* pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
|
|
@@ -284,6 +674,7 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
284
674
|
['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
|
|
285
675
|
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
286
676
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
677
|
+
['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
|
|
287
678
|
]);
|
|
288
679
|
|
|
289
680
|
/**
|
|
@@ -292,7 +683,7 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
292
683
|
* Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
|
|
293
684
|
* equivalent exists) or if no f16 activation shaders are present.
|
|
294
685
|
*
|
|
295
|
-
* NOTE: The caller is responsible for also updating
|
|
686
|
+
* NOTE: The caller is responsible for also updating session.activationDtype
|
|
296
687
|
* to reflect the widened dtype.
|
|
297
688
|
*
|
|
298
689
|
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
@@ -322,6 +713,12 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
322
713
|
const result = cloneGraph(graph);
|
|
323
714
|
|
|
324
715
|
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
716
|
+
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
717
|
+
?? deriveQ4PrefillF32ActivationKernelEntry(entry);
|
|
718
|
+
if (q4FallbackEntry) {
|
|
719
|
+
result.kernels[key] = q4FallbackEntry;
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
325
722
|
const replacement = shaderMap.get(entry.kernel);
|
|
326
723
|
if (replacement !== undefined) {
|
|
327
724
|
result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
|
|
@@ -331,6 +728,78 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
331
728
|
return result;
|
|
332
729
|
}
|
|
333
730
|
|
|
731
|
+
/**
|
|
732
|
+
* Widen an f16 execution graph onto the stable f32 correctness lane used for
|
|
733
|
+
* alternate-plan recovery after finiteness failure.
|
|
734
|
+
*
|
|
735
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
736
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
737
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
738
|
+
*/
|
|
739
|
+
export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
740
|
+
if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
|
|
741
|
+
return null;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
const hasTargetShader = Object.values(graph.kernels).some(
|
|
745
|
+
(entry) => F16_TO_F32_CORRECTNESS_FALLBACK_MAP.has(entry.kernel)
|
|
746
|
+
);
|
|
747
|
+
if (!hasTargetShader) {
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
const result = cloneGraph(graph);
|
|
752
|
+
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
753
|
+
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
754
|
+
?? deriveQ4PrefillF32ActivationKernelEntry(entry);
|
|
755
|
+
if (q4FallbackEntry) {
|
|
756
|
+
result.kernels[key] = q4FallbackEntry;
|
|
757
|
+
continue;
|
|
758
|
+
}
|
|
759
|
+
const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
|
|
760
|
+
if (replacement !== undefined) {
|
|
761
|
+
result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
return result;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Narrow f32-activation shaders back onto their f16-activation equivalents.
|
|
769
|
+
*
|
|
770
|
+
* Returns null if the graph has no supported f32-activation kernels to swap or
|
|
771
|
+
* if the runtime did not explicitly request f16 activations on an f16-capable
|
|
772
|
+
* GPU.
|
|
773
|
+
*
|
|
774
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
775
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
776
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
777
|
+
*/
|
|
778
|
+
export function narrowToF16Activations(graph, ctx) {
|
|
779
|
+
if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
780
|
+
return null;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
const hasTargetShader = Object.values(graph.kernels).some(
|
|
784
|
+
(entry) => !hasExplicitF32ActivationContract(entry) && F32_TO_F16_ACTIVATION_MAP.has(entry.kernel)
|
|
785
|
+
);
|
|
786
|
+
if (!hasTargetShader) {
|
|
787
|
+
return null;
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
const result = cloneGraph(graph);
|
|
791
|
+
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
792
|
+
if (hasExplicitF32ActivationContract(entry)) {
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
const replacement = F32_TO_F16_ACTIVATION_MAP.get(entry.kernel);
|
|
796
|
+
if (replacement !== undefined) {
|
|
797
|
+
result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return result;
|
|
801
|
+
}
|
|
802
|
+
|
|
334
803
|
// =============================================================================
|
|
335
804
|
// Transform: swapPrefillAttention
|
|
336
805
|
// =============================================================================
|
|
@@ -390,6 +859,30 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
390
859
|
return changed ? result : null;
|
|
391
860
|
}
|
|
392
861
|
|
|
862
|
+
// =============================================================================
|
|
863
|
+
// Transform: useHead256PrefillAttention
|
|
864
|
+
// =============================================================================
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* Promote prefill attention onto the fixed 256-dim shared-block kernel.
|
|
868
|
+
*
|
|
869
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
870
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
871
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
872
|
+
*/
|
|
873
|
+
export function useHead256PrefillAttention(graph, ctx) {
|
|
874
|
+
return (
|
|
875
|
+
swapPrefillAttention(graph, ctx, {
|
|
876
|
+
from: 'attention_small_f16kv.wgsl',
|
|
877
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
878
|
+
})
|
|
879
|
+
|| swapPrefillAttention(graph, ctx, {
|
|
880
|
+
from: 'attention_streaming_f16kv.wgsl',
|
|
881
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
882
|
+
})
|
|
883
|
+
);
|
|
884
|
+
}
|
|
885
|
+
|
|
393
886
|
// =============================================================================
|
|
394
887
|
// Transform: widenProjectionWeightsToF32
|
|
395
888
|
// =============================================================================
|
|
@@ -414,6 +907,19 @@ const LAYER_PROJECTION_OPS = new Set([
|
|
|
414
907
|
'gate_proj', 'up_proj', 'down_proj',
|
|
415
908
|
]);
|
|
416
909
|
|
|
910
|
+
const DENSE_Q4_PREFILL_FILES = new Set([
|
|
911
|
+
'matmul_f16w_f32a.wgsl',
|
|
912
|
+
'matmul_f16w_f32a_tiled.wgsl',
|
|
913
|
+
'matmul_f16.wgsl',
|
|
914
|
+
'matmul_f16_tiled.wgsl',
|
|
915
|
+
]);
|
|
916
|
+
|
|
917
|
+
function resolveDensePrefillProjectionKernel(ctx) {
|
|
918
|
+
return ctx.activationDtype === 'f16'
|
|
919
|
+
? 'matmul_f16.wgsl'
|
|
920
|
+
: 'matmul_f16w_f32a.wgsl';
|
|
921
|
+
}
|
|
922
|
+
|
|
417
923
|
/**
|
|
418
924
|
* Replace projection matmul kernels with f32 weight variants.
|
|
419
925
|
*
|
|
@@ -472,6 +978,700 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
472
978
|
return result;
|
|
473
979
|
}
|
|
474
980
|
|
|
981
|
+
// =============================================================================
|
|
982
|
+
// Transform: remapDenseQ4KPrefillToQ4Native
|
|
983
|
+
// =============================================================================
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* Replace dense prefill projection kernels with Q4-native prefill variants.
|
|
987
|
+
*
|
|
988
|
+
* This applies only when the graph already exposes a compatible fused Q4 decode
|
|
989
|
+
* projection kernel. All prefill layer projections are remapped to the shared-A
|
|
990
|
+
* batched multicol Q4 prefill kernel so the transformed path remains valid for
|
|
991
|
+
* `M > 1` prefill workloads.
|
|
992
|
+
*
|
|
993
|
+
* Returns null when the graph does not have the required dense-prefill + Q4
|
|
994
|
+
* decode shape.
|
|
995
|
+
*
|
|
996
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
997
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
998
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
999
|
+
*/
|
|
1000
|
+
export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
1001
|
+
const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
1002
|
+
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1003
|
+
return false;
|
|
1004
|
+
}
|
|
1005
|
+
const entry = graph.kernels[step[1]];
|
|
1006
|
+
return entry != null && DENSE_Q4_PREFILL_FILES.has(entry.kernel);
|
|
1007
|
+
});
|
|
1008
|
+
if (densePrefillProjectionSteps.length === 0) {
|
|
1009
|
+
return null;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
const result = cloneGraph(graph);
|
|
1013
|
+
const existingSharedKey = findKernelKeyByFileAndEntry(
|
|
1014
|
+
result,
|
|
1015
|
+
'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
1016
|
+
'main'
|
|
1017
|
+
);
|
|
1018
|
+
let sharedKey = existingSharedKey;
|
|
1019
|
+
if (!sharedKey) {
|
|
1020
|
+
const q4DecodeKey = findPhaseKernelKey(
|
|
1021
|
+
graph,
|
|
1022
|
+
graph.decode || [],
|
|
1023
|
+
LAYER_PROJECTION_OPS,
|
|
1024
|
+
(entry) => entry.kernel === 'fused_matmul_q4.wgsl'
|
|
1025
|
+
);
|
|
1026
|
+
if (!q4DecodeKey) {
|
|
1027
|
+
return null;
|
|
1028
|
+
}
|
|
1029
|
+
const q4DecodeEntry = result.kernels[q4DecodeKey];
|
|
1030
|
+
sharedKey = deriveKernelKey(result.kernels, q4DecodeKey, '_prefill_shared');
|
|
1031
|
+
result.kernels[sharedKey] = deriveKernelEntry(
|
|
1032
|
+
q4DecodeEntry,
|
|
1033
|
+
'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
1034
|
+
'main',
|
|
1035
|
+
null
|
|
1036
|
+
);
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
let changed = false;
|
|
1040
|
+
result.prefill = (result.prefill || []).map((step) => {
|
|
1041
|
+
const op = step[0];
|
|
1042
|
+
if (!LAYER_PROJECTION_OPS.has(op)) {
|
|
1043
|
+
return step;
|
|
1044
|
+
}
|
|
1045
|
+
const entry = result.kernels[step[1]];
|
|
1046
|
+
if (!entry || !DENSE_Q4_PREFILL_FILES.has(entry.kernel)) {
|
|
1047
|
+
return step;
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
const replacementKey = sharedKey;
|
|
1051
|
+
if (replacementKey === step[1]) {
|
|
1052
|
+
return step;
|
|
1053
|
+
}
|
|
1054
|
+
changed = true;
|
|
1055
|
+
const next = [...step];
|
|
1056
|
+
next[1] = replacementKey;
|
|
1057
|
+
return next;
|
|
1058
|
+
});
|
|
1059
|
+
|
|
1060
|
+
return changed ? result : null;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
// =============================================================================
|
|
1064
|
+
// Transform: remapQ4KPrefillToDense
|
|
1065
|
+
// =============================================================================
|
|
1066
|
+
|
|
1067
|
+
/**
|
|
1068
|
+
* Replace fused Q4K prefill projection kernels with dense tiled variants.
|
|
1069
|
+
*
|
|
1070
|
+
* Decode remains unchanged so the runtime can keep using fused Q4K decode while
|
|
1071
|
+
* the loader exposes mixed dense+Q4K materializations for prefill.
|
|
1072
|
+
*
|
|
1073
|
+
* Returns null when the graph has no fused Q4K prefill projection kernels.
|
|
1074
|
+
*
|
|
1075
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1076
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1077
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1078
|
+
*/
|
|
1079
|
+
export function remapQ4KPrefillToDense(graph, ctx) {
|
|
1080
|
+
const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
1081
|
+
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1082
|
+
return false;
|
|
1083
|
+
}
|
|
1084
|
+
const entry = graph.kernels[step[1]];
|
|
1085
|
+
return entry != null && entry.kernel.startsWith('fused_matmul_q4');
|
|
1086
|
+
});
|
|
1087
|
+
if (q4PrefillProjectionSteps.length === 0) {
|
|
1088
|
+
return null;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
const denseKernelFile = resolveDensePrefillProjectionKernel(ctx);
|
|
1092
|
+
const result = cloneGraph(graph);
|
|
1093
|
+
let denseKey = findKernelKeyByFileAndEntry(result, denseKernelFile, 'main');
|
|
1094
|
+
if (!denseKey) {
|
|
1095
|
+
const sourceKey = q4PrefillProjectionSteps[0][1];
|
|
1096
|
+
const sourceEntry = result.kernels[sourceKey];
|
|
1097
|
+
denseKey = deriveKernelKey(result.kernels, sourceKey, '_prefill_dense');
|
|
1098
|
+
result.kernels[denseKey] = deriveKernelEntry(
|
|
1099
|
+
sourceEntry,
|
|
1100
|
+
denseKernelFile,
|
|
1101
|
+
'main',
|
|
1102
|
+
null
|
|
1103
|
+
);
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
let changed = false;
|
|
1107
|
+
result.prefill = (result.prefill || []).map((step) => {
|
|
1108
|
+
if (!LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1109
|
+
return step;
|
|
1110
|
+
}
|
|
1111
|
+
const entry = result.kernels[step[1]];
|
|
1112
|
+
if (!entry || !entry.kernel.startsWith('fused_matmul_q4')) {
|
|
1113
|
+
return step;
|
|
1114
|
+
}
|
|
1115
|
+
if (step[1] === denseKey) {
|
|
1116
|
+
return step;
|
|
1117
|
+
}
|
|
1118
|
+
changed = true;
|
|
1119
|
+
const next = [...step];
|
|
1120
|
+
next[1] = denseKey;
|
|
1121
|
+
return next;
|
|
1122
|
+
});
|
|
1123
|
+
|
|
1124
|
+
return changed ? result : null;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// =============================================================================
|
|
1128
|
+
// Transform: useLinearDecodeProjectionF16
|
|
1129
|
+
// =============================================================================
|
|
1130
|
+
|
|
1131
|
+
/**
|
|
1132
|
+
* Remap the linear-attention q_proj decode step onto the f16-activation fused
|
|
1133
|
+
* Q4 kernel for linear-attention layers only. Full-attention layers keep the
|
|
1134
|
+
* manifest-wide f32 activation contract.
|
|
1135
|
+
*
|
|
1136
|
+
* Only q_proj is remapped. o_proj is intentionally excluded: the o_proj
|
|
1137
|
+
* output enters the residual stream directly, and f16 truncation there
|
|
1138
|
+
* accumulates across the 18 linear-attention layers in the Qwen 3.5 pattern,
|
|
1139
|
+
* corrupting the logit distribution (empirically verified: degenerate
|
|
1140
|
+
* repetitive output under greedy decode). q_proj f16 is safe because the
|
|
1141
|
+
* linear attention core absorbs the f16 input into its f32 internal state.
|
|
1142
|
+
*
|
|
1143
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1144
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1145
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1146
|
+
*/
|
|
1147
|
+
export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
1148
|
+
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
1149
|
+
if (!layerTypes || layerTypes.length === 0) {
|
|
1150
|
+
return null;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
const matchingLayers = layerTypes
|
|
1154
|
+
.map((layerType, layerIdx) => ({ layerType, layerIdx }))
|
|
1155
|
+
.filter(({ layerType }) => isLinearAttentionLayerType(layerType))
|
|
1156
|
+
.map(({ layerIdx }) => layerIdx);
|
|
1157
|
+
if (matchingLayers.length === 0) {
|
|
1158
|
+
return null;
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
const result = cloneGraph(graph);
|
|
1162
|
+
const targetLayers = {
|
|
1163
|
+
allLayers: layerTypes.map((_, layerIdx) => layerIdx),
|
|
1164
|
+
matchingLayers,
|
|
1165
|
+
};
|
|
1166
|
+
const qProjIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1167
|
+
if (qProjIndex === -1) {
|
|
1168
|
+
return null;
|
|
1169
|
+
}
|
|
1170
|
+
const qProjStep = result.decode[qProjIndex];
|
|
1171
|
+
const qProjKernelKey = qProjStep[1];
|
|
1172
|
+
const qProjKernel = result.kernels[qProjKernelKey];
|
|
1173
|
+
if (!qProjKernel) {
|
|
1174
|
+
return null;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
const derivedEntry = deriveLinearDecodeF16KernelEntry(qProjKernel);
|
|
1178
|
+
if (!derivedEntry) {
|
|
1179
|
+
return null;
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
const derivedKey = deriveKernelKey(result.kernels, qProjKernelKey, '_linear_f16out');
|
|
1183
|
+
result.kernels[derivedKey] = derivedEntry;
|
|
1184
|
+
const groupedEntries = buildGroupedLayerEntries(qProjStep, targetLayers, derivedKey);
|
|
1185
|
+
if (groupedEntries.length === 0) {
|
|
1186
|
+
return null;
|
|
1187
|
+
}
|
|
1188
|
+
result.decode = [
|
|
1189
|
+
...result.decode.slice(0, qProjIndex),
|
|
1190
|
+
...groupedEntries,
|
|
1191
|
+
...result.decode.slice(qProjIndex + 1),
|
|
1192
|
+
];
|
|
1193
|
+
|
|
1194
|
+
return result;
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// =============================================================================
|
|
1198
|
+
// Transform: remapQ4KDecodeToGemv
|
|
1199
|
+
// =============================================================================
|
|
1200
|
+
|
|
1201
|
+
/**
|
|
1202
|
+
* Replace fused Q4K decode projection kernels with GEMV subgroup variants.
|
|
1203
|
+
*
|
|
1204
|
+
* When Q4K weights have f16 materializations (mixed/dense loader mode), the
|
|
1205
|
+
* GEMV subgroup kernel on pre-dequantized f16 weights is significantly faster
|
|
1206
|
+
* than the fused Q4K kernel for M=1 decode (empirically 2.3x on Apple M-series).
|
|
1207
|
+
*
|
|
1208
|
+
* After this transform no decode kernels reference fused_matmul_q4*, which
|
|
1209
|
+
* signals the loader to use dense materialization (f16 only — no Q4K buffer
|
|
1210
|
+
* retained in GPU memory, reducing peak VRAM).
|
|
1211
|
+
*
|
|
1212
|
+
* Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
|
|
1213
|
+
* attention, residual, activation) are left untouched.
|
|
1214
|
+
*
|
|
1215
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1216
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1217
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1218
|
+
*/
|
|
1219
|
+
export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
1220
|
+
if (ctx.activationDtype === 'f16') {
|
|
1221
|
+
return null;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
const decodeSteps = graph.decode || [];
|
|
1225
|
+
const fusedDecodeKeys = new Set();
|
|
1226
|
+
for (const step of decodeSteps) {
|
|
1227
|
+
if (!Array.isArray(step)) continue;
|
|
1228
|
+
const kernelKey = step[1];
|
|
1229
|
+
const entry = graph.kernels[kernelKey];
|
|
1230
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
1231
|
+
fusedDecodeKeys.add(kernelKey);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
if (fusedDecodeKeys.size === 0) {
|
|
1235
|
+
return null;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
const result = cloneGraph(graph);
|
|
1239
|
+
const keyMap = new Map();
|
|
1240
|
+
|
|
1241
|
+
for (const key of fusedDecodeKeys) {
|
|
1242
|
+
const newKey = deriveKernelKey(result.kernels, key, '_gemv');
|
|
1243
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
1244
|
+
result.kernels[key],
|
|
1245
|
+
'matmul_gemv_subgroup.wgsl',
|
|
1246
|
+
'main_multicol',
|
|
1247
|
+
null
|
|
1248
|
+
);
|
|
1249
|
+
keyMap.set(key, newKey);
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
result.decode = remapStepKeys(result.decode, keyMap);
|
|
1253
|
+
return result;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
// =============================================================================
|
|
1257
|
+
// Transform: remapQ4KDecodeAttentionToGemv (diagnostic)
|
|
1258
|
+
// =============================================================================
|
|
1259
|
+
|
|
1260
|
+
const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
|
|
1261
|
+
|
|
1262
|
+
/**
|
|
1263
|
+
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
|
|
1264
|
+
* subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
|
|
1265
|
+
*
|
|
1266
|
+
* Diagnostic transform for isolating whether the GEMV correctness regression
|
|
1267
|
+
* originates in the attention or FFN projection path. Because FFN ops keep
|
|
1268
|
+
* their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
|
|
1269
|
+
* loader remains in mixed-materialization mode.
|
|
1270
|
+
*
|
|
1271
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1272
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1273
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1274
|
+
*/
|
|
1275
|
+
export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
1276
|
+
if (ctx.activationDtype === 'f16') {
|
|
1277
|
+
return null;
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
const decodeSteps = graph.decode || [];
|
|
1281
|
+
const attnFusedKeys = new Set();
|
|
1282
|
+
for (const step of decodeSteps) {
|
|
1283
|
+
if (!Array.isArray(step)) continue;
|
|
1284
|
+
const op = step[0];
|
|
1285
|
+
if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
|
|
1286
|
+
const kernelKey = step[1];
|
|
1287
|
+
const entry = graph.kernels[kernelKey];
|
|
1288
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
1289
|
+
attnFusedKeys.add(kernelKey);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
if (attnFusedKeys.size === 0) {
|
|
1293
|
+
return null;
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
const result = cloneGraph(graph);
|
|
1297
|
+
const keyMap = new Map();
|
|
1298
|
+
|
|
1299
|
+
for (const key of attnFusedKeys) {
|
|
1300
|
+
const newKey = deriveKernelKey(result.kernels, key, '_attn_gemv');
|
|
1301
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
1302
|
+
result.kernels[key],
|
|
1303
|
+
'matmul_gemv_subgroup.wgsl',
|
|
1304
|
+
'main_multicol',
|
|
1305
|
+
null
|
|
1306
|
+
);
|
|
1307
|
+
keyMap.set(key, newKey);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// Only remap attention projection steps, leave FFN steps unchanged.
|
|
1311
|
+
result.decode = result.decode.map((step) => {
|
|
1312
|
+
if (!Array.isArray(step)) return step;
|
|
1313
|
+
if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
|
|
1314
|
+
const replacement = keyMap.get(step[1]);
|
|
1315
|
+
if (replacement !== undefined) {
|
|
1316
|
+
const newStep = [...step];
|
|
1317
|
+
newStep[1] = replacement;
|
|
1318
|
+
return newStep;
|
|
1319
|
+
}
|
|
1320
|
+
return step;
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
return result;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
// =============================================================================
|
|
1327
|
+
// Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
|
|
1328
|
+
// =============================================================================
|
|
1329
|
+
|
|
1330
|
+
/**
|
|
1331
|
+
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
|
|
1332
|
+
* optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
|
|
1333
|
+
* cooperative loading with fast nibble extraction for maximum M=1 throughput
|
|
1334
|
+
* while preserving full Q4K dequant precision (no f16 weight materialization).
|
|
1335
|
+
*
|
|
1336
|
+
* This is the production fix for the f16-precision-loss regression observed
|
|
1337
|
+
* when attention projections use the f16-weight GEMV path: softmax amplifies
|
|
1338
|
+
* the f16 round-trip error in Q/K/V projections, causing garbage output.
|
|
1339
|
+
* By keeping inline Q4K dequant (f32 arithmetic) the attention path stays
|
|
1340
|
+
* numerically correct. FFN projections are unaffected and can safely use
|
|
1341
|
+
* the f16-weight GEMV path via remapQ4KDecodeFFNToGemv.
|
|
1342
|
+
*
|
|
1343
|
+
* Because the derived kernel still references fused_matmul_q4.wgsl,
|
|
1344
|
+
* isKernelPathFusedQ4K stays true and the weight loader remains in
|
|
1345
|
+
* mixed-materialization mode (Q4K retained for attention, f16 for FFN).
|
|
1346
|
+
*
|
|
1347
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1348
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1349
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1350
|
+
*/
|
|
1351
|
+
export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
1352
|
+
if (ctx.activationDtype === 'f16') {
|
|
1353
|
+
return null;
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
const decodeSteps = graph.decode || [];
|
|
1357
|
+
const attnFusedKeys = new Set();
|
|
1358
|
+
for (const step of decodeSteps) {
|
|
1359
|
+
if (!Array.isArray(step)) continue;
|
|
1360
|
+
const op = step[0];
|
|
1361
|
+
if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
|
|
1362
|
+
const kernelKey = step[1];
|
|
1363
|
+
const entry = graph.kernels[kernelKey];
|
|
1364
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
1365
|
+
attnFusedKeys.add(kernelKey);
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
if (attnFusedKeys.size === 0) {
|
|
1369
|
+
return null;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
const result = cloneGraph(graph);
|
|
1373
|
+
const keyMap = new Map();
|
|
1374
|
+
|
|
1375
|
+
for (const key of attnFusedKeys) {
|
|
1376
|
+
const newKey = deriveKernelKey(result.kernels, key, '_gemv');
|
|
1377
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
1378
|
+
result.kernels[key],
|
|
1379
|
+
'fused_matmul_q4.wgsl',
|
|
1380
|
+
'main_gemv',
|
|
1381
|
+
null
|
|
1382
|
+
);
|
|
1383
|
+
keyMap.set(key, newKey);
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
// Only remap attention projection steps, leave FFN steps unchanged.
|
|
1387
|
+
result.decode = result.decode.map((step) => {
|
|
1388
|
+
if (!Array.isArray(step)) return step;
|
|
1389
|
+
if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
|
|
1390
|
+
const replacement = keyMap.get(step[1]);
|
|
1391
|
+
if (replacement !== undefined) {
|
|
1392
|
+
const newStep = [...step];
|
|
1393
|
+
newStep[1] = replacement;
|
|
1394
|
+
return newStep;
|
|
1395
|
+
}
|
|
1396
|
+
return step;
|
|
1397
|
+
});
|
|
1398
|
+
|
|
1399
|
+
return result;
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
// =============================================================================
|
|
1403
|
+
// Transform: remapQ4KDecodeFFNToGemv (diagnostic)
|
|
1404
|
+
// =============================================================================
|
|
1405
|
+
|
|
1406
|
+
const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
1407
|
+
|
|
1408
|
+
/**
|
|
1409
|
+
* Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
|
|
1410
|
+
* variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
|
|
1411
|
+
*
|
|
1412
|
+
* Diagnostic complement to `remapQ4KDecodeAttentionToGemv`. Together these
|
|
1413
|
+
* two transforms isolate whether the GEMV decode regression originates in
|
|
1414
|
+
* the attention or FFN projection path. Because attention ops keep their
|
|
1415
|
+
* fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
|
|
1416
|
+
* remains in mixed-materialization mode.
|
|
1417
|
+
*
|
|
1418
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1419
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1420
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1421
|
+
*/
|
|
1422
|
+
export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
1423
|
+
if (ctx.activationDtype === 'f16') {
|
|
1424
|
+
return null;
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
const decodeSteps = graph.decode || [];
|
|
1428
|
+
const ffnFusedKeys = new Set();
|
|
1429
|
+
for (const step of decodeSteps) {
|
|
1430
|
+
if (!Array.isArray(step)) continue;
|
|
1431
|
+
const op = step[0];
|
|
1432
|
+
if (!FFN_PROJECTION_OPS.has(op)) continue;
|
|
1433
|
+
const kernelKey = step[1];
|
|
1434
|
+
const entry = graph.kernels[kernelKey];
|
|
1435
|
+
if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
|
|
1436
|
+
ffnFusedKeys.add(kernelKey);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
if (ffnFusedKeys.size === 0) {
|
|
1440
|
+
return null;
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
const result = cloneGraph(graph);
|
|
1444
|
+
const keyMap = new Map();
|
|
1445
|
+
|
|
1446
|
+
for (const key of ffnFusedKeys) {
|
|
1447
|
+
const newKey = deriveKernelKey(result.kernels, key, '_ffn_gemv');
|
|
1448
|
+
result.kernels[newKey] = deriveKernelEntry(
|
|
1449
|
+
result.kernels[key],
|
|
1450
|
+
'matmul_gemv_subgroup.wgsl',
|
|
1451
|
+
'main_multicol',
|
|
1452
|
+
null
|
|
1453
|
+
);
|
|
1454
|
+
keyMap.set(key, newKey);
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
// Only remap FFN projection steps, leave attention steps unchanged.
|
|
1458
|
+
result.decode = result.decode.map((step) => {
|
|
1459
|
+
if (!Array.isArray(step)) return step;
|
|
1460
|
+
if (!FFN_PROJECTION_OPS.has(step[0])) return step;
|
|
1461
|
+
const replacement = keyMap.get(step[1]);
|
|
1462
|
+
if (replacement !== undefined) {
|
|
1463
|
+
const newStep = [...step];
|
|
1464
|
+
newStep[1] = replacement;
|
|
1465
|
+
return newStep;
|
|
1466
|
+
}
|
|
1467
|
+
return step;
|
|
1468
|
+
});
|
|
1469
|
+
|
|
1470
|
+
return result;
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// =============================================================================
|
|
1474
|
+
// Transform: useQwenDecodeF16Matmuls
|
|
1475
|
+
// =============================================================================
|
|
1476
|
+
|
|
1477
|
+
/**
|
|
1478
|
+
* Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
|
|
1479
|
+
* kernels while keeping the manifest-wide f32 activation contract intact.
|
|
1480
|
+
*
|
|
1481
|
+
* This transform is intentionally selective:
|
|
1482
|
+
* - FFN gate/up decode matmuls switch to f16a so decode can bypass the slow
|
|
1483
|
+
* fused-q4k FFN path when capability policy opts in.
|
|
1484
|
+
* - LM head decode switches to the subgroup f16a GEMV path.
|
|
1485
|
+
*
|
|
1486
|
+
* FFN down remains on the f32-output contract so the layer residual path stays
|
|
1487
|
+
* numerically aligned with the manifest-owned activation dtype.
|
|
1488
|
+
*
|
|
1489
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1490
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1491
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1492
|
+
*/
|
|
1493
|
+
export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
1494
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1495
|
+
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
|
|
1496
|
+
return null;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
const result = cloneGraph(graph);
|
|
1500
|
+
let changed = false;
|
|
1501
|
+
|
|
1502
|
+
for (const op of ['gate_proj', 'up_proj']) {
|
|
1503
|
+
const stepIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1504
|
+
if (stepIndex === -1) {
|
|
1505
|
+
continue;
|
|
1506
|
+
}
|
|
1507
|
+
const step = result.decode[stepIndex];
|
|
1508
|
+
const kernelKey = step[1];
|
|
1509
|
+
const kernelEntry = result.kernels[kernelKey];
|
|
1510
|
+
if (!kernelEntry) {
|
|
1511
|
+
continue;
|
|
1512
|
+
}
|
|
1513
|
+
const derivedEntry = deriveLinearDecodeF16KernelEntry(kernelEntry);
|
|
1514
|
+
if (!derivedEntry) {
|
|
1515
|
+
continue;
|
|
1516
|
+
}
|
|
1517
|
+
const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_decode_f16out');
|
|
1518
|
+
result.kernels[derivedKey] = derivedEntry;
|
|
1519
|
+
const replacement = [...step];
|
|
1520
|
+
replacement[1] = derivedKey;
|
|
1521
|
+
result.decode = [
|
|
1522
|
+
...result.decode.slice(0, stepIndex),
|
|
1523
|
+
replacement,
|
|
1524
|
+
...result.decode.slice(stepIndex + 1),
|
|
1525
|
+
];
|
|
1526
|
+
changed = true;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
const postLayerResult = replacePhaseStepKernelKey(
|
|
1530
|
+
result.postLayer ?? [],
|
|
1531
|
+
'lm_head',
|
|
1532
|
+
(() => {
|
|
1533
|
+
const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
|
|
1534
|
+
if (!lmHeadStep) {
|
|
1535
|
+
return null;
|
|
1536
|
+
}
|
|
1537
|
+
const lmHeadKernelKey = lmHeadStep[1];
|
|
1538
|
+
const lmHeadKernel = result.kernels[lmHeadKernelKey];
|
|
1539
|
+
if (!lmHeadKernel) {
|
|
1540
|
+
return null;
|
|
1541
|
+
}
|
|
1542
|
+
const derivedEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
|
|
1543
|
+
if (!derivedEntry) {
|
|
1544
|
+
return null;
|
|
1545
|
+
}
|
|
1546
|
+
const derivedKey = deriveKernelKey(result.kernels, lmHeadKernelKey, '_decode_f16out');
|
|
1547
|
+
result.kernels[derivedKey] = derivedEntry;
|
|
1548
|
+
return derivedKey;
|
|
1549
|
+
})()
|
|
1550
|
+
);
|
|
1551
|
+
if (postLayerResult.changed) {
|
|
1552
|
+
result.postLayer = postLayerResult.steps;
|
|
1553
|
+
changed = true;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
return changed ? result : null;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
// =============================================================================
|
|
1560
|
+
// Transform: useQwenF16PrimaryMatmuls
|
|
1561
|
+
// =============================================================================
|
|
1562
|
+
|
|
1563
|
+
/**
|
|
1564
|
+
* Promote the Qwen 3.5 0.8B execution graph onto its fast f16 primary lane
|
|
1565
|
+
* when the runtime explicitly requests f16 activations.
|
|
1566
|
+
*
|
|
1567
|
+
* This transform narrows the hot matmul path while keeping `o_proj` on the
|
|
1568
|
+
* stable manifest-owned f32-output kernel. The residual stream feeds directly
|
|
1569
|
+
* into the next RMSNorm, and the Qwen 3.5 0.8B promoted f16 lane becomes
|
|
1570
|
+
* numerically unstable when `o_proj` writes f16 there (observed first at the
|
|
1571
|
+
* first full-attention block's post-attention RMSNorm).
|
|
1572
|
+
*
|
|
1573
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1574
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1575
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1576
|
+
*/
|
|
1577
|
+
export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
1578
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1579
|
+
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
1580
|
+
return null;
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
1584
|
+
if (!layerTypes || layerTypes.length === 0) {
|
|
1585
|
+
return null;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
const result = cloneGraph(graph);
|
|
1589
|
+
let changed = false;
|
|
1590
|
+
|
|
1591
|
+
for (const [phaseName, op] of [['decode', 'attention'], ['prefill', 'attention']]) {
|
|
1592
|
+
const phaseSteps = result[phaseName] || [];
|
|
1593
|
+
const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1594
|
+
const kernelKey = step?.[1];
|
|
1595
|
+
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1596
|
+
const derivedEntry = deriveF16AttentionKernelEntry(kernelEntry);
|
|
1597
|
+
if (!step || !kernelKey || !derivedEntry) {
|
|
1598
|
+
continue;
|
|
1599
|
+
}
|
|
1600
|
+
const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f16');
|
|
1601
|
+
result.kernels[derivedKey] = derivedEntry;
|
|
1602
|
+
const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, derivedKey);
|
|
1603
|
+
if (phaseResult.changed) {
|
|
1604
|
+
result[phaseName] = phaseResult.steps;
|
|
1605
|
+
changed = true;
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1610
|
+
const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
|
|
1611
|
+
const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(decodeProjectionKernel);
|
|
1612
|
+
if (decodeProjectionStep && decodeProjectionEntry) {
|
|
1613
|
+
const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_primary_f16');
|
|
1614
|
+
result.kernels[decodeProjectionKey] = decodeProjectionEntry;
|
|
1615
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj', 'gate_proj', 'up_proj']) {
|
|
1616
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
|
|
1617
|
+
if (phaseResult.changed) {
|
|
1618
|
+
result.decode = phaseResult.steps;
|
|
1619
|
+
changed = true;
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
const prefillProjectionStep = (result.prefill || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1625
|
+
const prefillProjectionKernel = prefillProjectionStep ? result.kernels[prefillProjectionStep[1]] : null;
|
|
1626
|
+
const prefillProjectionEntry = deriveQ4PrefillF16KernelEntry(prefillProjectionKernel);
|
|
1627
|
+
if (prefillProjectionStep && prefillProjectionEntry) {
|
|
1628
|
+
const prefillProjectionKey = deriveKernelKey(result.kernels, prefillProjectionStep[1], '_primary_f16');
|
|
1629
|
+
result.kernels[prefillProjectionKey] = prefillProjectionEntry;
|
|
1630
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj']) {
|
|
1631
|
+
const phaseResult = replacePhaseStepKernelKey(result.prefill, op, prefillProjectionKey);
|
|
1632
|
+
if (phaseResult.changed) {
|
|
1633
|
+
result.prefill = phaseResult.steps;
|
|
1634
|
+
changed = true;
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
for (const [phaseName, op] of [['decode', 'o_proj'], ['prefill', 'o_proj']]) {
|
|
1640
|
+
const phaseSteps = result[phaseName] || [];
|
|
1641
|
+
const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1642
|
+
const kernelKey = step?.[1];
|
|
1643
|
+
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1644
|
+
if (!step || !kernelKey || !kernelEntry) {
|
|
1645
|
+
continue;
|
|
1646
|
+
}
|
|
1647
|
+
const boundaryKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f32_boundary');
|
|
1648
|
+
result.kernels[boundaryKey] = deriveKernelEntryWithPrecision(kernelEntry, {
|
|
1649
|
+
inputDtype: 'f32',
|
|
1650
|
+
outputDtype: 'f32',
|
|
1651
|
+
});
|
|
1652
|
+
const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, boundaryKey);
|
|
1653
|
+
if (phaseResult.changed) {
|
|
1654
|
+
result[phaseName] = phaseResult.steps;
|
|
1655
|
+
changed = true;
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
|
|
1660
|
+
const lmHeadKernel = lmHeadStep ? result.kernels[lmHeadStep[1]] : null;
|
|
1661
|
+
const lmHeadEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
|
|
1662
|
+
if (lmHeadStep && lmHeadEntry) {
|
|
1663
|
+
const lmHeadKey = deriveKernelKey(result.kernels, lmHeadStep[1], '_primary_f16');
|
|
1664
|
+
result.kernels[lmHeadKey] = lmHeadEntry;
|
|
1665
|
+
const phaseResult = replacePhaseStepKernelKey(result.postLayer, 'lm_head', lmHeadKey);
|
|
1666
|
+
if (phaseResult.changed) {
|
|
1667
|
+
result.postLayer = phaseResult.steps;
|
|
1668
|
+
changed = true;
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
return changed ? result : null;
|
|
1673
|
+
}
|
|
1674
|
+
|
|
475
1675
|
// =============================================================================
|
|
476
1676
|
// Composition
|
|
477
1677
|
// =============================================================================
|
|
@@ -504,9 +1704,21 @@ export function composeTransforms(...transforms) {
|
|
|
504
1704
|
|
|
505
1705
|
/** @type {Readonly<Record<string, Function>>} */
|
|
506
1706
|
export const TRANSFORMS = Object.freeze({
|
|
1707
|
+
narrowToF16Activations,
|
|
507
1708
|
removeSubgroups,
|
|
508
1709
|
widenToF32Activations,
|
|
1710
|
+
widenToF32CorrectnessFallback,
|
|
509
1711
|
swapPrefillAttention,
|
|
1712
|
+
useHead256PrefillAttention,
|
|
510
1713
|
widenProjectionWeightsToF32,
|
|
1714
|
+
remapDenseQ4KPrefillToQ4Native,
|
|
1715
|
+
remapQ4KPrefillToDense,
|
|
1716
|
+
useLinearDecodeProjectionF16,
|
|
1717
|
+
remapQ4KDecodeToGemv,
|
|
1718
|
+
remapQ4KDecodeAttentionToGemv,
|
|
1719
|
+
remapQ4KDecodeAttentionToFusedQ4KGemv,
|
|
1720
|
+
remapQ4KDecodeFFNToGemv,
|
|
1721
|
+
useQwenF16PrimaryMatmuls,
|
|
1722
|
+
useQwenDecodeF16Matmuls,
|
|
511
1723
|
composeTransforms,
|
|
512
1724
|
});
|