doppler-gpu 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/README.md +63 -38
- package/assets/doppler.svg +57 -1601
- package/package.json +111 -32
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +486 -0
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +885 -744
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +469 -43
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +66 -10
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +69 -4
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +34 -50
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +35 -2
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +148 -3
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +8 -60
- package/src/client/wrap-pipeline-handle.d.ts +82 -0
- package/src/client/wrap-pipeline-handle.js +259 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +28 -157
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +163 -126
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +3593 -1146
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +3 -1
- package/src/config/merge-helpers.js +2 -1
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +65 -7
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +20 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +16 -0
- package/src/config/schema/converter.schema.js +11 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +13 -2
- package/src/config/schema/doppler.schema.js +140 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +39 -1
- package/src/config/schema/execution-v1.schema.js +35 -20
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +14 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +175 -2
- package/src/config/schema/manifest.schema.js +58 -3
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -9
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +183 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
- package/src/config/transforms/execution-graph-transforms.js +829 -168
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +31 -14
- package/src/converter/core.d.ts +65 -0
- package/src/converter/core.js +1153 -22
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +218 -5
- package/src/converter/quantizer.d.ts +24 -4
- package/src/converter/quantizer.js +92 -2
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +296 -38
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1733 -58
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +5 -14
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +192 -12
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +111 -2
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +10 -0
- package/src/formats/rdrr/index.js +10 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +26 -6
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +193 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +111 -2
- package/src/formats/rdrr/validation.js +260 -1
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +45 -23
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +130 -14
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +1 -8
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +458 -16
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +13 -120
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +84 -8
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +25 -22
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +77 -5
- package/src/gpu/kernels/index.js +55 -4
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +44 -2
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +172 -10
- package/src/gpu/kernels/matmul.d.ts +1 -29
- package/src/gpu/kernels/matmul.js +240 -38
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +1 -56
- package/src/gpu/kernels/moe.js +46 -33
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -12
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +6 -4
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +30 -7
- package/src/gpu/kernels/rmsnorm.js +49 -8
- package/src/gpu/kernels/rmsnorm.wgsl +15 -12
- package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +28 -21
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +8 -223
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +43 -0
- package/src/gpu/kernels/utils.js +81 -11
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +10 -12
- package/src/gpu/uniform-cache.js +25 -30
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
- package/src/inference/browser-harness-suite-helpers.js +525 -3
- package/src/inference/browser-harness-text-helpers.d.ts +89 -0
- package/src/inference/browser-harness-text-helpers.js +459 -25
- package/src/inference/browser-harness.d.ts +7 -28
- package/src/inference/browser-harness.js +556 -79
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +31 -0
- package/src/inference/kv-cache/base.js +125 -2
- package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
- package/src/inference/kv-cache/mixed-geometry.js +121 -4
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
- package/src/inference/pipelines/text/attention/projections.js +398 -71
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +542 -56
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +343 -50
- package/src/inference/pipelines/text/attention/types.d.ts +7 -1
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +25 -26
- package/src/inference/pipelines/text/config.js +304 -25
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +25 -29
- package/src/inference/pipelines/text/embed.js +16 -45
- package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
- package/src/inference/pipelines/text/execution-plan.js +21 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
- package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
- package/src/inference/pipelines/text/execution-v1.js +461 -158
- package/src/inference/pipelines/text/ffn/dense.js +214 -6
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +84 -17
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +9 -7
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
- package/src/inference/pipelines/text/generator-steps.js +275 -79
- package/src/inference/pipelines/text/generator.d.ts +38 -5
- package/src/inference/pipelines/text/generator.js +1259 -496
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +13 -3
- package/src/inference/pipelines/text/init.js +213 -725
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +373 -558
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +419 -66
- package/src/inference/pipelines/text/logits/index.d.ts +3 -3
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +42 -4
- package/src/inference/pipelines/text/ops.js +20 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +119 -31
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +153 -9
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +19 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
- package/src/inference/pipelines/text/types.d.ts +92 -81
- package/src/inference/pipelines/text/weights.d.ts +4 -52
- package/src/inference/pipelines/text/weights.js +6 -56
- package/src/inference/pipelines/text.d.ts +33 -1
- package/src/inference/pipelines/text.js +375 -55
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +341 -174
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +14 -0
- package/src/loader/doppler-loader.js +378 -25
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +276 -11
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +61 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +52 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +36 -8
- package/src/loader/tensors/tensor-loader.js +521 -22
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +13 -42
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +57 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +162 -10
- package/src/rules/inference/capability-transforms.rules.json +111 -0
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +117 -16
- package/src/rules/inference/ffn.rules.json +24 -2
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +17 -0
- package/src/rules/rule-registry.js +59 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +241 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +31 -4
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +53 -9
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +8 -3
- package/src/tooling/command-api.d.ts +22 -1
- package/src/tooling/command-api.js +11 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +14 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +3 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +571 -37
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +107 -17
- package/src/tooling/node-command-runner.js +43 -2
- package/src/tooling/node-convert-worker-pool.js +15 -0
- package/src/tooling/node-convert-worker.js +11 -1
- package/src/tooling/node-converter.js +102 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +112 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +10 -5
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +110 -8
- package/src/tooling/source-runtime-bundle.d.ts +3 -0
- package/src/tooling/source-runtime-bundle.js +30 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +25 -12
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -13,16 +13,14 @@
|
|
|
13
13
|
// Helpers
|
|
14
14
|
// =============================================================================
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
/*
|
|
17
17
|
* Deep-clone an execution graph.
|
|
18
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
19
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph}
|
|
20
18
|
*/
|
|
21
19
|
function cloneGraph(graph) {
|
|
22
20
|
return structuredClone(graph);
|
|
23
21
|
}
|
|
24
22
|
|
|
25
|
-
|
|
23
|
+
/*
|
|
26
24
|
* Shader files that require subgroups even though "subgroup" is not in the filename.
|
|
27
25
|
* Online attention kernels use subgroup reductions internally.
|
|
28
26
|
*/
|
|
@@ -35,21 +33,24 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
|
35
33
|
['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
36
34
|
['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
37
35
|
['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
36
|
+
['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
38
37
|
['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
39
38
|
['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
39
|
+
['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
40
40
|
['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
41
41
|
['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
42
42
|
['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
43
|
+
['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
43
44
|
['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
44
45
|
['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
45
46
|
['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
46
47
|
['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
48
|
+
['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
49
|
+
['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
47
50
|
]);
|
|
48
51
|
|
|
49
|
-
|
|
52
|
+
/*
|
|
50
53
|
* Check whether a kernel entry requires subgroup support.
|
|
51
|
-
* @param {{ kernel: string }} kernelEntry
|
|
52
|
-
* @returns {boolean}
|
|
53
54
|
*/
|
|
54
55
|
function isSubgroupKernel(kernelEntry) {
|
|
55
56
|
if (typeof kernelEntry.kernel !== 'string') return false;
|
|
@@ -61,11 +62,8 @@ function requiresNoSubgroupFallback(kernelEntry) {
|
|
|
61
62
|
return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
|
|
65
|
+
/*
|
|
65
66
|
* Find all kernel keys in the graph whose `kernel` file matches the given filename.
|
|
66
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
67
|
-
* @param {string} filename
|
|
68
|
-
* @returns {string[]}
|
|
69
67
|
*/
|
|
70
68
|
function findKernelKeysByFile(graph, filename) {
|
|
71
69
|
const keys = [];
|
|
@@ -77,23 +75,15 @@ function findKernelKeysByFile(graph, filename) {
|
|
|
77
75
|
return keys;
|
|
78
76
|
}
|
|
79
77
|
|
|
80
|
-
|
|
78
|
+
/*
|
|
81
79
|
* Check whether any kernel in the graph uses the given shader file.
|
|
82
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
83
|
-
* @param {string} filename
|
|
84
|
-
* @returns {boolean}
|
|
85
80
|
*/
|
|
86
81
|
function hasKernelFile(graph, filename) {
|
|
87
82
|
return findKernelKeysByFile(graph, filename).length > 0;
|
|
88
83
|
}
|
|
89
84
|
|
|
90
|
-
|
|
85
|
+
/*
|
|
91
86
|
* Create a new kernel entry with the digest cleared (shader changed).
|
|
92
|
-
* @param {object} base - original kernel entry
|
|
93
|
-
* @param {string} newFile - new shader filename
|
|
94
|
-
* @param {string} newEntry - new entry point name
|
|
95
|
-
* @param {object|null} [constants] - override constants (null to remove)
|
|
96
|
-
* @returns {object}
|
|
97
87
|
*/
|
|
98
88
|
function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
99
89
|
const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
|
|
@@ -111,6 +101,11 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
|
111
101
|
return derived;
|
|
112
102
|
}
|
|
113
103
|
|
|
104
|
+
function deriveMappedKernelEntry(base, newFile) {
|
|
105
|
+
const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
|
|
106
|
+
return deriveKernelEntry(base, newFile, newEntry);
|
|
107
|
+
}
|
|
108
|
+
|
|
114
109
|
function deriveKernelPrecision(base, newFile) {
|
|
115
110
|
const precision = base.precision ? { ...base.precision } : {};
|
|
116
111
|
const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
|
|
@@ -135,12 +130,8 @@ function deriveKernelPrecision(base, newFile) {
|
|
|
135
130
|
return precision;
|
|
136
131
|
}
|
|
137
132
|
|
|
138
|
-
|
|
133
|
+
/*
|
|
139
134
|
* Derive a non-colliding kernel key name.
|
|
140
|
-
* @param {object} kernels - existing kernels dict
|
|
141
|
-
* @param {string} baseKey - original key
|
|
142
|
-
* @param {string} suffix - suffix to append
|
|
143
|
-
* @returns {string}
|
|
144
135
|
*/
|
|
145
136
|
function deriveKernelKey(kernels, baseKey, suffix) {
|
|
146
137
|
const candidate = `${baseKey}${suffix}`;
|
|
@@ -154,11 +145,8 @@ function deriveKernelKey(kernels, baseKey, suffix) {
|
|
|
154
145
|
return `${candidate}_${counter}`;
|
|
155
146
|
}
|
|
156
147
|
|
|
157
|
-
|
|
148
|
+
/*
|
|
158
149
|
* Replace kernel key references in step tuples.
|
|
159
|
-
* @param {Array<Array>} steps
|
|
160
|
-
* @param {Map<string, string>} keyMap - oldKey → newKey
|
|
161
|
-
* @returns {Array<Array>}
|
|
162
150
|
*/
|
|
163
151
|
function remapStepKeys(steps, keyMap) {
|
|
164
152
|
return steps.map((step) => {
|
|
@@ -173,12 +161,8 @@ function remapStepKeys(steps, keyMap) {
|
|
|
173
161
|
});
|
|
174
162
|
}
|
|
175
163
|
|
|
176
|
-
|
|
164
|
+
/*
|
|
177
165
|
* Check whether a step tuple's kernel key resolves to the given shader file.
|
|
178
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
179
|
-
* @param {Array} step
|
|
180
|
-
* @param {string} filename
|
|
181
|
-
* @returns {boolean}
|
|
182
166
|
*/
|
|
183
167
|
function stepUsesFile(graph, step, filename) {
|
|
184
168
|
const kernelKey = step[1];
|
|
@@ -186,14 +170,9 @@ function stepUsesFile(graph, step, filename) {
|
|
|
186
170
|
return entry != null && entry.kernel === filename;
|
|
187
171
|
}
|
|
188
172
|
|
|
189
|
-
|
|
173
|
+
/*
|
|
190
174
|
* Find the first kernel key used by matching ops in a phase whose shader file
|
|
191
175
|
* satisfies the provided predicate.
|
|
192
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
193
|
-
* @param {Array<Array>} steps
|
|
194
|
-
* @param {ReadonlySet<string>} ops
|
|
195
|
-
* @param {(entry: { kernel: string, entry: string }) => boolean} predicate
|
|
196
|
-
* @returns {string | null}
|
|
197
176
|
*/
|
|
198
177
|
function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
199
178
|
for (const step of steps || []) {
|
|
@@ -208,12 +187,8 @@ function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
|
208
187
|
return null;
|
|
209
188
|
}
|
|
210
189
|
|
|
211
|
-
|
|
190
|
+
/*
|
|
212
191
|
* Find an existing kernel key by shader file and entry point.
|
|
213
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
214
|
-
* @param {string} filename
|
|
215
|
-
* @param {string} entryPoint
|
|
216
|
-
* @returns {string | null}
|
|
217
192
|
*/
|
|
218
193
|
function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
|
|
219
194
|
for (const [key, entry] of Object.entries(graph.kernels)) {
|
|
@@ -292,6 +267,28 @@ function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
|
|
|
292
267
|
return { steps: nextSteps, changed };
|
|
293
268
|
}
|
|
294
269
|
|
|
270
|
+
function findPhaseStep(steps, op) {
|
|
271
|
+
if (!Array.isArray(steps) || !op) {
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
for (const entry of steps) {
|
|
275
|
+
if (Array.isArray(entry)) {
|
|
276
|
+
if (entry[0] === op) {
|
|
277
|
+
return entry;
|
|
278
|
+
}
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const nested = findPhaseStep(entry.steps, op);
|
|
285
|
+
if (nested) {
|
|
286
|
+
return nested;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
|
|
295
292
|
function deriveKernelEntryWithPrecision(base, precision) {
|
|
296
293
|
return {
|
|
297
294
|
...base,
|
|
@@ -307,6 +304,7 @@ const ATTENTION_F16KV_TO_F16_MAP = new Map([
|
|
|
307
304
|
['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
|
|
308
305
|
['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
|
|
309
306
|
['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
|
|
307
|
+
['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
|
|
310
308
|
]);
|
|
311
309
|
|
|
312
310
|
function deriveF16AttentionKernelEntry(base) {
|
|
@@ -366,6 +364,29 @@ function deriveLmHeadDecodeF16KernelEntry(base) {
|
|
|
366
364
|
return null;
|
|
367
365
|
}
|
|
368
366
|
|
|
367
|
+
function deriveDenseDecodeF16KernelEntry(base) {
|
|
368
|
+
if (typeof base?.kernel !== 'string') {
|
|
369
|
+
return null;
|
|
370
|
+
}
|
|
371
|
+
const precision = {
|
|
372
|
+
inputDtype: 'f16',
|
|
373
|
+
outputDtype: 'f16',
|
|
374
|
+
};
|
|
375
|
+
if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
|
|
376
|
+
return {
|
|
377
|
+
...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
|
|
378
|
+
precision: {
|
|
379
|
+
...(base.precision ?? {}),
|
|
380
|
+
...precision,
|
|
381
|
+
},
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
|
|
385
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
386
|
+
}
|
|
387
|
+
return null;
|
|
388
|
+
}
|
|
389
|
+
|
|
369
390
|
function deriveQ4DecodeF16KernelEntry(base) {
|
|
370
391
|
if (typeof base?.kernel !== 'string') {
|
|
371
392
|
return null;
|
|
@@ -400,6 +421,9 @@ function deriveQ4PrefillF16KernelEntry(base) {
|
|
|
400
421
|
inputDtype: 'f16',
|
|
401
422
|
outputDtype: 'f16',
|
|
402
423
|
};
|
|
424
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
425
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
426
|
+
}
|
|
403
427
|
if (base.kernel.startsWith('fused_matmul_q4_batched')) {
|
|
404
428
|
return {
|
|
405
429
|
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
|
|
@@ -412,6 +436,53 @@ function deriveQ4PrefillF16KernelEntry(base) {
|
|
|
412
436
|
return null;
|
|
413
437
|
}
|
|
414
438
|
|
|
439
|
+
function deriveQ4WideTilePrefillF16KernelEntry(base) {
|
|
440
|
+
if (typeof base?.kernel !== 'string') {
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
const precision = {
|
|
444
|
+
inputDtype: 'f16',
|
|
445
|
+
outputDtype: 'f16',
|
|
446
|
+
};
|
|
447
|
+
if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
|
|
448
|
+
return {
|
|
449
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
|
|
450
|
+
precision: {
|
|
451
|
+
...(base.precision ?? {}),
|
|
452
|
+
...precision,
|
|
453
|
+
},
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
457
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
458
|
+
}
|
|
459
|
+
return null;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function deriveQ4PrefillF16AccumKernelEntry(base) {
|
|
463
|
+
if (typeof base?.kernel !== 'string') {
|
|
464
|
+
return null;
|
|
465
|
+
}
|
|
466
|
+
const precision = {
|
|
467
|
+
inputDtype: 'f16',
|
|
468
|
+
outputDtype: 'f16',
|
|
469
|
+
};
|
|
470
|
+
if (
|
|
471
|
+
base.kernel === 'fused_matmul_q4_widetile.wgsl'
|
|
472
|
+
|| base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
473
|
+
|| base.kernel.startsWith('fused_matmul_q4_batched')
|
|
474
|
+
) {
|
|
475
|
+
return {
|
|
476
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
|
|
477
|
+
precision: {
|
|
478
|
+
...(base.precision ?? {}),
|
|
479
|
+
...precision,
|
|
480
|
+
},
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
return null;
|
|
484
|
+
}
|
|
485
|
+
|
|
415
486
|
function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
416
487
|
if (typeof base?.kernel !== 'string') {
|
|
417
488
|
return null;
|
|
@@ -438,16 +509,38 @@ function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
|
438
509
|
return null;
|
|
439
510
|
}
|
|
440
511
|
|
|
441
|
-
function deriveQ4PrefillF32ActivationKernelEntry(base) {
|
|
512
|
+
function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
|
|
442
513
|
if (typeof base?.kernel !== 'string') {
|
|
443
514
|
return null;
|
|
444
515
|
}
|
|
516
|
+
const replacement = options.fullF32 === true
|
|
517
|
+
? {
|
|
518
|
+
kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
519
|
+
entry: 'main',
|
|
520
|
+
}
|
|
521
|
+
: {
|
|
522
|
+
kernel: 'fused_matmul_q4_widetile.wgsl',
|
|
523
|
+
entry: 'main',
|
|
524
|
+
};
|
|
525
|
+
if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
|
|
526
|
+
return deriveKernelEntryWithPrecision(
|
|
527
|
+
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
528
|
+
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
529
|
+
);
|
|
530
|
+
}
|
|
445
531
|
if (
|
|
446
532
|
base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
|
|
447
533
|
&& base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
|
|
534
|
+
&& base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
448
535
|
) {
|
|
449
536
|
return null;
|
|
450
537
|
}
|
|
538
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
539
|
+
return deriveKernelEntryWithPrecision(
|
|
540
|
+
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
541
|
+
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
542
|
+
);
|
|
543
|
+
}
|
|
451
544
|
return deriveKernelEntryWithPrecision(
|
|
452
545
|
deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
|
|
453
546
|
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
@@ -476,15 +569,12 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
|
|
|
476
569
|
// Transform: removeSubgroups
|
|
477
570
|
// =============================================================================
|
|
478
571
|
|
|
479
|
-
|
|
572
|
+
/*
|
|
480
573
|
* Remove subgroup shader dependencies from decode and postLayer steps.
|
|
481
574
|
* Prefill steps are left untouched (they already use tiled matmul).
|
|
482
575
|
*
|
|
483
576
|
* Returns null if the graph has no subgroup kernels.
|
|
484
577
|
*
|
|
485
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
486
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
487
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
488
578
|
*/
|
|
489
579
|
export function removeSubgroups(graph, ctx) {
|
|
490
580
|
const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
|
|
@@ -571,19 +661,21 @@ export function removeSubgroups(graph, ctx) {
|
|
|
571
661
|
// Transform: widenToF32Activations
|
|
572
662
|
// =============================================================================
|
|
573
663
|
|
|
574
|
-
|
|
664
|
+
/*
|
|
575
665
|
* Activation-only widening: f16-activation shaders → f32-activation variants
|
|
576
666
|
* that still use f16 for weights and KV cache. Requires shader-f16 for weight
|
|
577
667
|
* and KV buffer reads.
|
|
578
|
-
* @type {ReadonlyMap<string, string>}
|
|
579
668
|
*/
|
|
580
669
|
const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
581
670
|
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
582
671
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
583
672
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
584
673
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
674
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
585
675
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
586
676
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
677
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
678
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
587
679
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
588
680
|
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
589
681
|
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
@@ -591,20 +683,25 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
591
683
|
['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
|
|
592
684
|
['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
|
|
593
685
|
['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
686
|
+
['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
|
|
594
687
|
]);
|
|
595
688
|
|
|
596
|
-
|
|
689
|
+
/*
|
|
597
690
|
* Activation-only narrowing: f32-activation shaders that still consume f16
|
|
598
691
|
* weights/KV are rewritten onto the matching f16-activation lane.
|
|
599
692
|
*
|
|
600
693
|
* This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
|
|
601
694
|
* runtime session explicitly requests f16 activations for an execution-v1
|
|
602
695
|
* graph that was authored with conservative f32 activation defaults.
|
|
603
|
-
* @type {ReadonlyMap<string, string>}
|
|
604
696
|
*/
|
|
605
697
|
const F32_TO_F16_ACTIVATION_MAP = new Map(
|
|
606
698
|
Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
|
|
607
699
|
);
|
|
700
|
+
F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
|
|
701
|
+
F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
|
|
702
|
+
// head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
|
|
703
|
+
// f16 requests fail-closed until that path has its own evidence.
|
|
704
|
+
F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
|
|
608
705
|
|
|
609
706
|
function hasExplicitF32ActivationContract(entry) {
|
|
610
707
|
const precision = entry?.precision;
|
|
@@ -616,37 +713,38 @@ function hasExplicitF32ActivationContract(entry) {
|
|
|
616
713
|
|| precision.outputDtype === 'f32';
|
|
617
714
|
}
|
|
618
715
|
|
|
619
|
-
|
|
716
|
+
/*
|
|
620
717
|
* Correctness fallback: preserve f16 weights where possible, but widen both
|
|
621
718
|
* activations and KV-cache interactions onto the stable f32 execution lane.
|
|
622
719
|
* Used for alternate-plan recovery after finiteness failure.
|
|
623
|
-
* @type {ReadonlyMap<string, string>}
|
|
624
720
|
*/
|
|
625
721
|
const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
626
722
|
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
627
723
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
628
724
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
629
725
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
726
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
630
727
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
631
728
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
729
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
730
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
632
731
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
633
732
|
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
634
733
|
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
635
|
-
['attention_decode_online_f16.wgsl', '
|
|
636
|
-
['attention_decode_chunked_f16.wgsl', '
|
|
734
|
+
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
735
|
+
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
637
736
|
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
638
737
|
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
639
|
-
['attention_decode_online_f16kv.wgsl', '
|
|
640
|
-
['attention_decode_chunked_f16kv.wgsl', '
|
|
738
|
+
['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
739
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
641
740
|
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
642
741
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
643
742
|
]);
|
|
644
743
|
|
|
645
|
-
|
|
744
|
+
/*
|
|
646
745
|
* Full f32 widening: every shader that uses `enable f16;` is replaced with a
|
|
647
746
|
* pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
|
|
648
747
|
* Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
|
|
649
|
-
* @type {ReadonlyMap<string, string>}
|
|
650
748
|
*/
|
|
651
749
|
const FULL_F32_SHADER_MAP = new Map([
|
|
652
750
|
// f16-activation utility kernels → f32
|
|
@@ -654,8 +752,11 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
654
752
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
655
753
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
656
754
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
755
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
657
756
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
658
757
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
758
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
759
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
659
760
|
// f16-activation matmul → f32
|
|
660
761
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
|
|
661
762
|
['matmul_f16.wgsl', 'matmul_f32.wgsl'],
|
|
@@ -665,19 +766,19 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
665
766
|
['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
|
|
666
767
|
['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
|
|
667
768
|
// f16-activation attention → f32
|
|
668
|
-
['attention_decode_online_f16.wgsl', '
|
|
669
|
-
['attention_decode_chunked_f16.wgsl', '
|
|
769
|
+
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
770
|
+
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
670
771
|
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
671
772
|
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
672
773
|
// f16kv attention (f32 Q, f16 KV) → f32
|
|
673
|
-
['attention_decode_online_f16kv.wgsl', '
|
|
674
|
-
['attention_decode_chunked_f16kv.wgsl', '
|
|
774
|
+
['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
775
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
675
776
|
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
676
777
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
677
778
|
['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
|
|
678
779
|
]);
|
|
679
780
|
|
|
680
|
-
|
|
781
|
+
/*
|
|
681
782
|
* Widen all f16-activation shaders to f32-activation equivalents.
|
|
682
783
|
*
|
|
683
784
|
* Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
|
|
@@ -686,9 +787,6 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
686
787
|
* NOTE: The caller is responsible for also updating session.activationDtype
|
|
687
788
|
* to reflect the widened dtype.
|
|
688
789
|
*
|
|
689
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
690
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
691
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
692
790
|
*/
|
|
693
791
|
export function widenToF32Activations(graph, ctx) {
|
|
694
792
|
// Bail out if fused f16 FFN is present — no direct f32 equivalent
|
|
@@ -702,9 +800,12 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
702
800
|
const shaderMap = ctx.capabilities?.hasF16 === false
|
|
703
801
|
? FULL_F32_SHADER_MAP
|
|
704
802
|
: F16_TO_F32_ACTIVATION_MAP;
|
|
803
|
+
const fullF32 = ctx.capabilities?.hasF16 === false;
|
|
705
804
|
|
|
706
805
|
const hasTargetShader = Object.values(graph.kernels).some(
|
|
707
806
|
(entry) => shaderMap.has(entry.kernel)
|
|
807
|
+
|| deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
808
|
+
|| deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
|
|
708
809
|
);
|
|
709
810
|
if (!hasTargetShader) {
|
|
710
811
|
return null;
|
|
@@ -714,27 +815,24 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
714
815
|
|
|
715
816
|
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
716
817
|
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
717
|
-
?? deriveQ4PrefillF32ActivationKernelEntry(entry);
|
|
818
|
+
?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
|
|
718
819
|
if (q4FallbackEntry) {
|
|
719
820
|
result.kernels[key] = q4FallbackEntry;
|
|
720
821
|
continue;
|
|
721
822
|
}
|
|
722
823
|
const replacement = shaderMap.get(entry.kernel);
|
|
723
824
|
if (replacement !== undefined) {
|
|
724
|
-
result.kernels[key] =
|
|
825
|
+
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
725
826
|
}
|
|
726
827
|
}
|
|
727
828
|
|
|
728
829
|
return result;
|
|
729
830
|
}
|
|
730
831
|
|
|
731
|
-
|
|
832
|
+
/*
|
|
732
833
|
* Widen an f16 execution graph onto the stable f32 correctness lane used for
|
|
733
834
|
* alternate-plan recovery after finiteness failure.
|
|
734
835
|
*
|
|
735
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
736
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
737
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
738
836
|
*/
|
|
739
837
|
export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
740
838
|
if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
|
|
@@ -758,22 +856,19 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
|
758
856
|
}
|
|
759
857
|
const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
|
|
760
858
|
if (replacement !== undefined) {
|
|
761
|
-
result.kernels[key] =
|
|
859
|
+
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
762
860
|
}
|
|
763
861
|
}
|
|
764
862
|
return result;
|
|
765
863
|
}
|
|
766
864
|
|
|
767
|
-
|
|
865
|
+
/*
|
|
768
866
|
* Narrow f32-activation shaders back onto their f16-activation equivalents.
|
|
769
867
|
*
|
|
770
868
|
* Returns null if the graph has no supported f32-activation kernels to swap or
|
|
771
869
|
* if the runtime did not explicitly request f16 activations on an f16-capable
|
|
772
870
|
* GPU.
|
|
773
871
|
*
|
|
774
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
775
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
776
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
777
872
|
*/
|
|
778
873
|
export function narrowToF16Activations(graph, ctx) {
|
|
779
874
|
if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
@@ -804,7 +899,6 @@ export function narrowToF16Activations(graph, ctx) {
|
|
|
804
899
|
// Transform: swapPrefillAttention
|
|
805
900
|
// =============================================================================
|
|
806
901
|
|
|
807
|
-
/** @type {ReadonlyMap<string, string>} */
|
|
808
902
|
const PREFILL_ATTENTION_PAIRS = new Map([
|
|
809
903
|
['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
|
|
810
904
|
['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
@@ -812,7 +906,27 @@ const PREFILL_ATTENTION_PAIRS = new Map([
|
|
|
812
906
|
['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
|
|
813
907
|
]);
|
|
814
908
|
|
|
815
|
-
|
|
909
|
+
function graphUsesKernelKeyInPrefill(graph, kernelKey) {
|
|
910
|
+
for (const entry of graph.prefill || []) {
|
|
911
|
+
if (Array.isArray(entry)) {
|
|
912
|
+
if (entry[1] === kernelKey) {
|
|
913
|
+
return true;
|
|
914
|
+
}
|
|
915
|
+
continue;
|
|
916
|
+
}
|
|
917
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
for (const step of entry.steps) {
|
|
921
|
+
if (Array.isArray(step) && step[1] === kernelKey) {
|
|
922
|
+
return true;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
return false;
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/*
|
|
816
930
|
* Swap prefill attention kernel between streaming and small variants.
|
|
817
931
|
*
|
|
818
932
|
* The `opts` parameter specifies the direction:
|
|
@@ -821,10 +935,6 @@ const PREFILL_ATTENTION_PAIRS = new Map([
|
|
|
821
935
|
* If `from`/`to` are not provided, uses the bidirectional pair map.
|
|
822
936
|
* Returns null if no matching prefill attention kernel is found.
|
|
823
937
|
*
|
|
824
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
825
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
826
|
-
* @param {{ from?: string, to?: string }} [opts]
|
|
827
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
828
938
|
*/
|
|
829
939
|
export function swapPrefillAttention(graph, ctx, opts) {
|
|
830
940
|
const from = opts?.from;
|
|
@@ -847,8 +957,7 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
847
957
|
}
|
|
848
958
|
|
|
849
959
|
if (target !== undefined) {
|
|
850
|
-
|
|
851
|
-
const usedInPrefill = (graph.prefill || []).some((step) => step[1] === key);
|
|
960
|
+
const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
|
|
852
961
|
if (usedInPrefill) {
|
|
853
962
|
result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
|
|
854
963
|
changed = true;
|
|
@@ -863,31 +972,47 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
863
972
|
// Transform: useHead256PrefillAttention
|
|
864
973
|
// =============================================================================
|
|
865
974
|
|
|
866
|
-
|
|
975
|
+
/*
|
|
976
|
+
* Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
|
|
977
|
+
*
|
|
978
|
+
*/
|
|
979
|
+
export function useHead256SmallPrefillAttention(graph, ctx) {
|
|
980
|
+
return swapPrefillAttention(graph, ctx, {
|
|
981
|
+
from: 'attention_small_f16kv.wgsl',
|
|
982
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/*
|
|
867
987
|
* Promote prefill attention onto the fixed 256-dim shared-block kernel.
|
|
868
988
|
*
|
|
869
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
870
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
871
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
872
989
|
*/
|
|
873
990
|
export function useHead256PrefillAttention(graph, ctx) {
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
991
|
+
let current = graph;
|
|
992
|
+
let changed = false;
|
|
993
|
+
|
|
994
|
+
const smallRemap = useHead256SmallPrefillAttention(current, ctx);
|
|
995
|
+
if (smallRemap) {
|
|
996
|
+
current = smallRemap;
|
|
997
|
+
changed = true;
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
const streamingRemap = swapPrefillAttention(current, ctx, {
|
|
1001
|
+
from: 'attention_streaming_f16kv.wgsl',
|
|
1002
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
1003
|
+
});
|
|
1004
|
+
if (streamingRemap) {
|
|
1005
|
+
current = streamingRemap;
|
|
1006
|
+
changed = true;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
return changed ? current : null;
|
|
884
1010
|
}
|
|
885
1011
|
|
|
886
1012
|
// =============================================================================
|
|
887
1013
|
// Transform: widenProjectionWeightsToF32
|
|
888
1014
|
// =============================================================================
|
|
889
1015
|
|
|
890
|
-
/** @type {ReadonlySet<string>} */
|
|
891
1016
|
const PROJECTION_MATMUL_FILES = new Set([
|
|
892
1017
|
'matmul_gemv_subgroup.wgsl',
|
|
893
1018
|
'matmul_gemv_subgroup_f16a.wgsl',
|
|
@@ -897,10 +1022,9 @@ const PROJECTION_MATMUL_FILES = new Set([
|
|
|
897
1022
|
'matmul_f16_tiled.wgsl',
|
|
898
1023
|
]);
|
|
899
1024
|
|
|
900
|
-
|
|
1025
|
+
/*
|
|
901
1026
|
* Known layer projection ops. Only these are widened; lm_head and embed are
|
|
902
1027
|
* excluded.
|
|
903
|
-
* @type {ReadonlySet<string>}
|
|
904
1028
|
*/
|
|
905
1029
|
const LAYER_PROJECTION_OPS = new Set([
|
|
906
1030
|
'q_proj', 'k_proj', 'v_proj', 'o_proj',
|
|
@@ -920,7 +1044,7 @@ function resolveDensePrefillProjectionKernel(ctx) {
|
|
|
920
1044
|
: 'matmul_f16w_f32a.wgsl';
|
|
921
1045
|
}
|
|
922
1046
|
|
|
923
|
-
|
|
1047
|
+
/*
|
|
924
1048
|
* Replace projection matmul kernels with f32 weight variants.
|
|
925
1049
|
*
|
|
926
1050
|
* Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
|
|
@@ -928,9 +1052,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
|
|
|
928
1052
|
*
|
|
929
1053
|
* Returns null if no applicable projection kernels are found.
|
|
930
1054
|
*
|
|
931
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
932
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
933
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
934
1055
|
*/
|
|
935
1056
|
export function widenProjectionWeightsToF32(graph, ctx) {
|
|
936
1057
|
// Collect kernel keys used by layer projection steps across all phases
|
|
@@ -982,7 +1103,7 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
982
1103
|
// Transform: remapDenseQ4KPrefillToQ4Native
|
|
983
1104
|
// =============================================================================
|
|
984
1105
|
|
|
985
|
-
|
|
1106
|
+
/*
|
|
986
1107
|
* Replace dense prefill projection kernels with Q4-native prefill variants.
|
|
987
1108
|
*
|
|
988
1109
|
* This applies only when the graph already exposes a compatible fused Q4 decode
|
|
@@ -993,9 +1114,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
993
1114
|
* Returns null when the graph does not have the required dense-prefill + Q4
|
|
994
1115
|
* decode shape.
|
|
995
1116
|
*
|
|
996
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
997
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
998
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
999
1117
|
*/
|
|
1000
1118
|
export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
1001
1119
|
const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
@@ -1064,7 +1182,7 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
|
1064
1182
|
// Transform: remapQ4KPrefillToDense
|
|
1065
1183
|
// =============================================================================
|
|
1066
1184
|
|
|
1067
|
-
|
|
1185
|
+
/*
|
|
1068
1186
|
* Replace fused Q4K prefill projection kernels with dense tiled variants.
|
|
1069
1187
|
*
|
|
1070
1188
|
* Decode remains unchanged so the runtime can keep using fused Q4K decode while
|
|
@@ -1072,9 +1190,6 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
|
1072
1190
|
*
|
|
1073
1191
|
* Returns null when the graph has no fused Q4K prefill projection kernels.
|
|
1074
1192
|
*
|
|
1075
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1076
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1077
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1078
1193
|
*/
|
|
1079
1194
|
export function remapQ4KPrefillToDense(graph, ctx) {
|
|
1080
1195
|
const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
@@ -1128,7 +1243,7 @@ export function remapQ4KPrefillToDense(graph, ctx) {
|
|
|
1128
1243
|
// Transform: useLinearDecodeProjectionF16
|
|
1129
1244
|
// =============================================================================
|
|
1130
1245
|
|
|
1131
|
-
|
|
1246
|
+
/*
|
|
1132
1247
|
* Remap the linear-attention q_proj decode step onto the f16-activation fused
|
|
1133
1248
|
* Q4 kernel for linear-attention layers only. Full-attention layers keep the
|
|
1134
1249
|
* manifest-wide f32 activation contract.
|
|
@@ -1140,9 +1255,6 @@ export function remapQ4KPrefillToDense(graph, ctx) {
|
|
|
1140
1255
|
* repetitive output under greedy decode). q_proj f16 is safe because the
|
|
1141
1256
|
* linear attention core absorbs the f16 input into its f32 internal state.
|
|
1142
1257
|
*
|
|
1143
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1144
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1145
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1146
1258
|
*/
|
|
1147
1259
|
export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
1148
1260
|
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
@@ -1198,7 +1310,7 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
|
1198
1310
|
// Transform: remapQ4KDecodeToGemv
|
|
1199
1311
|
// =============================================================================
|
|
1200
1312
|
|
|
1201
|
-
|
|
1313
|
+
/*
|
|
1202
1314
|
* Replace fused Q4K decode projection kernels with GEMV subgroup variants.
|
|
1203
1315
|
*
|
|
1204
1316
|
* When Q4K weights have f16 materializations (mixed/dense loader mode), the
|
|
@@ -1212,9 +1324,6 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
|
1212
1324
|
* Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
|
|
1213
1325
|
* attention, residual, activation) are left untouched.
|
|
1214
1326
|
*
|
|
1215
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1216
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1217
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1218
1327
|
*/
|
|
1219
1328
|
export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
1220
1329
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1259,7 +1368,7 @@ export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
|
1259
1368
|
|
|
1260
1369
|
const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
|
|
1261
1370
|
|
|
1262
|
-
|
|
1371
|
+
/*
|
|
1263
1372
|
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
|
|
1264
1373
|
* subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
|
|
1265
1374
|
*
|
|
@@ -1268,9 +1377,6 @@ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj'
|
|
|
1268
1377
|
* their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
|
|
1269
1378
|
* loader remains in mixed-materialization mode.
|
|
1270
1379
|
*
|
|
1271
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1272
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1273
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1274
1380
|
*/
|
|
1275
1381
|
export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
1276
1382
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1327,7 +1433,7 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
|
1327
1433
|
// Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
|
|
1328
1434
|
// =============================================================================
|
|
1329
1435
|
|
|
1330
|
-
|
|
1436
|
+
/*
|
|
1331
1437
|
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
|
|
1332
1438
|
* optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
|
|
1333
1439
|
* cooperative loading with fast nibble extraction for maximum M=1 throughput
|
|
@@ -1344,9 +1450,6 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
|
1344
1450
|
* isKernelPathFusedQ4K stays true and the weight loader remains in
|
|
1345
1451
|
* mixed-materialization mode (Q4K retained for attention, f16 for FFN).
|
|
1346
1452
|
*
|
|
1347
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1348
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1349
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1350
1453
|
*/
|
|
1351
1454
|
export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
1352
1455
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1405,7 +1508,7 @@ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
|
1405
1508
|
|
|
1406
1509
|
const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
1407
1510
|
|
|
1408
|
-
|
|
1511
|
+
/*
|
|
1409
1512
|
* Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
|
|
1410
1513
|
* variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
|
|
1411
1514
|
*
|
|
@@ -1415,9 +1518,6 @@ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
|
1415
1518
|
* fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
|
|
1416
1519
|
* remains in mixed-materialization mode.
|
|
1417
1520
|
*
|
|
1418
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1419
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1420
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1421
1521
|
*/
|
|
1422
1522
|
export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
1423
1523
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1474,7 +1574,7 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
|
1474
1574
|
// Transform: useQwenDecodeF16Matmuls
|
|
1475
1575
|
// =============================================================================
|
|
1476
1576
|
|
|
1477
|
-
|
|
1577
|
+
/*
|
|
1478
1578
|
* Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
|
|
1479
1579
|
* kernels while keeping the manifest-wide f32 activation contract intact.
|
|
1480
1580
|
*
|
|
@@ -1486,16 +1586,8 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
|
1486
1586
|
* FFN down remains on the f32-output contract so the layer residual path stays
|
|
1487
1587
|
* numerically aligned with the manifest-owned activation dtype.
|
|
1488
1588
|
*
|
|
1489
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1490
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1491
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1492
1589
|
*/
|
|
1493
1590
|
export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
1494
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1495
|
-
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
|
|
1496
|
-
return null;
|
|
1497
|
-
}
|
|
1498
|
-
|
|
1499
1591
|
const result = cloneGraph(graph);
|
|
1500
1592
|
let changed = false;
|
|
1501
1593
|
|
|
@@ -1560,26 +1652,20 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
|
1560
1652
|
// Transform: useQwenF16PrimaryMatmuls
|
|
1561
1653
|
// =============================================================================
|
|
1562
1654
|
|
|
1563
|
-
|
|
1564
|
-
* Promote the Qwen 3.5 0.8B execution graph onto its
|
|
1565
|
-
* when the runtime explicitly requests f16 activations.
|
|
1655
|
+
/*
|
|
1656
|
+
* Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
|
|
1657
|
+
* lane when the runtime explicitly requests f16 activations.
|
|
1566
1658
|
*
|
|
1567
|
-
* This transform narrows the
|
|
1568
|
-
* stable manifest-owned f32-output kernel. The
|
|
1569
|
-
*
|
|
1570
|
-
*
|
|
1571
|
-
*
|
|
1659
|
+
* This transform narrows the decode projection and LM-head path while keeping
|
|
1660
|
+
* `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
|
|
1661
|
+
* already authored in the manifest on WideTile/head256 kernels and remains
|
|
1662
|
+
* manifest-owned. The residual stream feeds directly into the next RMSNorm, and
|
|
1663
|
+
* the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
|
|
1664
|
+
* `o_proj` writes f16 there (observed first at the first full-attention block's
|
|
1665
|
+
* post-attention RMSNorm).
|
|
1572
1666
|
*
|
|
1573
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1574
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1575
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1576
1667
|
*/
|
|
1577
1668
|
export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
1578
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1579
|
-
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
1580
|
-
return null;
|
|
1581
|
-
}
|
|
1582
|
-
|
|
1583
1669
|
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
1584
1670
|
if (!layerTypes || layerTypes.length === 0) {
|
|
1585
1671
|
return null;
|
|
@@ -1672,18 +1758,560 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
|
1672
1758
|
return changed ? result : null;
|
|
1673
1759
|
}
|
|
1674
1760
|
|
|
1761
|
+
// =============================================================================
|
|
1762
|
+
// Transform: useQwen36F16Activations
|
|
1763
|
+
// =============================================================================
|
|
1764
|
+
|
|
1765
|
+
/*
|
|
1766
|
+
* Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
|
|
1767
|
+
*
|
|
1768
|
+
*/
|
|
1769
|
+
export function useQwen36F16Activations(graph, ctx) {
|
|
1770
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
1771
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
1772
|
+
let changed = narrowed != null;
|
|
1773
|
+
|
|
1774
|
+
const replaceKernelEntry = (key, entry) => {
|
|
1775
|
+
if (!key || !entry) {
|
|
1776
|
+
return;
|
|
1777
|
+
}
|
|
1778
|
+
result.kernels[key] = entry;
|
|
1779
|
+
changed = true;
|
|
1780
|
+
};
|
|
1781
|
+
const replaceOps = (phaseName, ops, kernelKey) => {
|
|
1782
|
+
for (const op of ops) {
|
|
1783
|
+
const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
|
|
1784
|
+
if (phaseResult.changed) {
|
|
1785
|
+
result[phaseName] = phaseResult.steps;
|
|
1786
|
+
changed = true;
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
};
|
|
1790
|
+
|
|
1791
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
1792
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
1793
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
1794
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
1795
|
+
replaceKernelEntry(
|
|
1796
|
+
embedKey,
|
|
1797
|
+
deriveKernelEntryWithPrecision(
|
|
1798
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
1799
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1800
|
+
)
|
|
1801
|
+
);
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
1805
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
1806
|
+
const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
|
|
1807
|
+
if (decodeProjectionEntry) {
|
|
1808
|
+
const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
|
|
1809
|
+
result.kernels[f16Key] = decodeProjectionEntry;
|
|
1810
|
+
replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
1814
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
1815
|
+
const prefillProjectionEntry =
|
|
1816
|
+
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
1817
|
+
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
1818
|
+
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
|
|
1819
|
+
if (prefillProjectionEntry) {
|
|
1820
|
+
const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
|
|
1821
|
+
result.kernels[f16Key] = prefillProjectionEntry;
|
|
1822
|
+
replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
1826
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
1827
|
+
replaceKernelEntry(
|
|
1828
|
+
lmHeadKey,
|
|
1829
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
1830
|
+
);
|
|
1831
|
+
|
|
1832
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
1833
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
1834
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
1835
|
+
const lmHeadPrefillF16Entry =
|
|
1836
|
+
deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
|
|
1837
|
+
?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
|
|
1838
|
+
?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
|
|
1839
|
+
if (lmHeadPrefillF16Entry) {
|
|
1840
|
+
replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
|
|
1841
|
+
} else if (
|
|
1842
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
1843
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
1844
|
+
) {
|
|
1845
|
+
replaceKernelEntry(
|
|
1846
|
+
lmHeadPrefillKey,
|
|
1847
|
+
deriveKernelEntryWithPrecision(
|
|
1848
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
1849
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1850
|
+
)
|
|
1851
|
+
);
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
return changed ? result : null;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
// =============================================================================
|
|
1858
|
+
// Transform: useGemma4Int4PleSelectiveF16Decode
|
|
1859
|
+
// =============================================================================
|
|
1860
|
+
|
|
1861
|
+
/*
|
|
1862
|
+
* Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
|
|
1863
|
+
* explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
|
|
1864
|
+
* attention because the repository does not currently have pure-f16 head256 or
|
|
1865
|
+
* head512 prefill kernels.
|
|
1866
|
+
*
|
|
1867
|
+
*/
|
|
1868
|
+
export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
|
|
1869
|
+
const result = cloneGraph(graph);
|
|
1870
|
+
let changed = false;
|
|
1871
|
+
|
|
1872
|
+
const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1873
|
+
const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
|
|
1874
|
+
const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
|
|
1875
|
+
if (decodeProjectionStep && decodeProjectionEntry) {
|
|
1876
|
+
const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
|
|
1877
|
+
result.kernels[decodeProjectionKey] = decodeProjectionEntry;
|
|
1878
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj']) {
|
|
1879
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
|
|
1880
|
+
if (phaseResult.changed) {
|
|
1881
|
+
result.decode = phaseResult.steps;
|
|
1882
|
+
changed = true;
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
for (const op of ['rope_q', 'rope_k']) {
|
|
1888
|
+
const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1889
|
+
const kernelKey = step?.[1];
|
|
1890
|
+
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1891
|
+
const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
|
|
1892
|
+
if (!step || !kernelKey || !replacement) {
|
|
1893
|
+
continue;
|
|
1894
|
+
}
|
|
1895
|
+
const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
|
|
1896
|
+
result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
|
|
1897
|
+
deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
|
|
1898
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1899
|
+
);
|
|
1900
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
|
|
1901
|
+
if (phaseResult.changed) {
|
|
1902
|
+
result.decode = phaseResult.steps;
|
|
1903
|
+
changed = true;
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
|
|
1908
|
+
const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
|
|
1909
|
+
const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
|
|
1910
|
+
if (attentionStep && attentionEntry) {
|
|
1911
|
+
const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
|
|
1912
|
+
result.kernels[attentionKey] = attentionEntry;
|
|
1913
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
|
|
1914
|
+
if (phaseResult.changed) {
|
|
1915
|
+
result.decode = phaseResult.steps;
|
|
1916
|
+
changed = true;
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
|
|
1921
|
+
const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
|
|
1922
|
+
if (oProjStep && oProjKernel) {
|
|
1923
|
+
const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
|
|
1924
|
+
result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
|
|
1925
|
+
inputDtype: 'f32',
|
|
1926
|
+
outputDtype: 'f32',
|
|
1927
|
+
});
|
|
1928
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
|
|
1929
|
+
if (phaseResult.changed) {
|
|
1930
|
+
result.decode = phaseResult.steps;
|
|
1931
|
+
changed = true;
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
return changed ? result : null;
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
// =============================================================================
|
|
1939
|
+
// Transform: useGemma4TextF16Activations
|
|
1940
|
+
// =============================================================================
|
|
1941
|
+
|
|
1942
|
+
const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
|
|
1943
|
+
'q_proj',
|
|
1944
|
+
'k_proj',
|
|
1945
|
+
'v_proj',
|
|
1946
|
+
'gate_proj',
|
|
1947
|
+
'up_proj',
|
|
1948
|
+
]);
|
|
1949
|
+
|
|
1950
|
+
function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
|
|
1951
|
+
if (!Array.isArray(entries) || !f16Key || !f32Key) {
|
|
1952
|
+
return { entries, changed: false };
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
let changed = false;
|
|
1956
|
+
const remapStep = (step) => {
|
|
1957
|
+
if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1958
|
+
return step;
|
|
1959
|
+
}
|
|
1960
|
+
const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
|
|
1961
|
+
? f32Key
|
|
1962
|
+
: f16Key;
|
|
1963
|
+
if (step[1] === targetKey) {
|
|
1964
|
+
return step;
|
|
1965
|
+
}
|
|
1966
|
+
const replacement = [...step];
|
|
1967
|
+
replacement[1] = targetKey;
|
|
1968
|
+
changed = true;
|
|
1969
|
+
return replacement;
|
|
1970
|
+
};
|
|
1971
|
+
|
|
1972
|
+
const nextEntries = entries.map((entry) => {
|
|
1973
|
+
if (Array.isArray(entry)) {
|
|
1974
|
+
return remapStep(entry);
|
|
1975
|
+
}
|
|
1976
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
1977
|
+
return entry;
|
|
1978
|
+
}
|
|
1979
|
+
return {
|
|
1980
|
+
...entry,
|
|
1981
|
+
steps: entry.steps.map((step) => remapStep(step)),
|
|
1982
|
+
};
|
|
1983
|
+
});
|
|
1984
|
+
|
|
1985
|
+
return { entries: nextEntries, changed };
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
|
|
1989
|
+
let changed = false;
|
|
1990
|
+
|
|
1991
|
+
const replaceStepWithSourceEntry = (phaseName, op, precision) => {
|
|
1992
|
+
const phase = result[phaseName];
|
|
1993
|
+
const sourcePhase = sourceGraph[phaseName];
|
|
1994
|
+
const step = findPhaseStep(phase, op);
|
|
1995
|
+
const sourceStep = findPhaseStep(sourcePhase, op);
|
|
1996
|
+
const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
|
|
1997
|
+
const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
|
|
1998
|
+
if (!step || !sourceKey || !sourceEntry) {
|
|
1999
|
+
return;
|
|
2000
|
+
}
|
|
2001
|
+
const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
|
|
2002
|
+
result.kernels[stableKey] = precision
|
|
2003
|
+
? deriveKernelEntryWithPrecision(sourceEntry, precision)
|
|
2004
|
+
: { ...sourceEntry };
|
|
2005
|
+
const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
|
|
2006
|
+
if (phaseResult.changed) {
|
|
2007
|
+
result[phaseName] = phaseResult.steps;
|
|
2008
|
+
changed = true;
|
|
2009
|
+
}
|
|
2010
|
+
};
|
|
2011
|
+
|
|
2012
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
|
|
2013
|
+
replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
|
|
2014
|
+
}
|
|
2015
|
+
replaceStepWithSourceEntry('decode', 'attention', {
|
|
2016
|
+
activationDtype: 'f32',
|
|
2017
|
+
kvDtype: 'f16',
|
|
2018
|
+
outputDtype: 'f32',
|
|
2019
|
+
});
|
|
2020
|
+
for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
|
|
2021
|
+
replaceStepWithSourceEntry('postLayer', op, null);
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
return changed;
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
|
|
2028
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2029
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
2030
|
+
let changed = narrowed != null;
|
|
2031
|
+
|
|
2032
|
+
const replaceKernelEntry = (key, entry) => {
|
|
2033
|
+
if (!key || !entry) {
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
result.kernels[key] = entry;
|
|
2037
|
+
changed = true;
|
|
2038
|
+
};
|
|
2039
|
+
const stableTextBoundary = options?.stableTextBoundary === true;
|
|
2040
|
+
|
|
2041
|
+
const derivePrefillAttentionEntry = (entry) => {
|
|
2042
|
+
if (
|
|
2043
|
+
stableTextBoundary
|
|
2044
|
+
&& typeof entry?.kernel === 'string'
|
|
2045
|
+
&& entry.kernel === 'attention_small_f16.wgsl'
|
|
2046
|
+
) {
|
|
2047
|
+
return deriveKernelEntryWithPrecision(
|
|
2048
|
+
deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
|
|
2049
|
+
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2050
|
+
);
|
|
2051
|
+
}
|
|
2052
|
+
if (
|
|
2053
|
+
stableTextBoundary
|
|
2054
|
+
&& typeof entry?.kernel === 'string'
|
|
2055
|
+
&& entry.kernel.endsWith('_f16kv.wgsl')
|
|
2056
|
+
) {
|
|
2057
|
+
return deriveKernelEntryWithPrecision(
|
|
2058
|
+
entry,
|
|
2059
|
+
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2060
|
+
);
|
|
2061
|
+
}
|
|
2062
|
+
return deriveF16AttentionKernelEntry(entry);
|
|
2063
|
+
};
|
|
2064
|
+
|
|
2065
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2066
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
2067
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2068
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2069
|
+
replaceKernelEntry(
|
|
2070
|
+
embedKey,
|
|
2071
|
+
deriveKernelEntryWithPrecision(
|
|
2072
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2073
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2074
|
+
)
|
|
2075
|
+
);
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2079
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2080
|
+
replaceKernelEntry(
|
|
2081
|
+
decodeProjectionKey,
|
|
2082
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2083
|
+
);
|
|
2084
|
+
|
|
2085
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2086
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2087
|
+
const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
|
|
2088
|
+
const prefillProjectionF16Entry = stableTextBoundary
|
|
2089
|
+
? (
|
|
2090
|
+
deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2091
|
+
?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2092
|
+
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2093
|
+
)
|
|
2094
|
+
: (
|
|
2095
|
+
deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2096
|
+
?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2097
|
+
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2098
|
+
);
|
|
2099
|
+
if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
|
|
2100
|
+
const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
|
|
2101
|
+
result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
|
|
2102
|
+
changed = true;
|
|
2103
|
+
const remapped = remapGemma412BPrefillProjectionEntries(
|
|
2104
|
+
result.prefill,
|
|
2105
|
+
prefillProjectionF16Key,
|
|
2106
|
+
prefillProjectionKey
|
|
2107
|
+
);
|
|
2108
|
+
result.prefill = remapped.entries;
|
|
2109
|
+
changed = changed || remapped.changed;
|
|
2110
|
+
} else {
|
|
2111
|
+
replaceKernelEntry(
|
|
2112
|
+
prefillProjectionKey,
|
|
2113
|
+
prefillProjectionF16Entry
|
|
2114
|
+
);
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
const replacePrefillAttentionEntries = (entries) => {
|
|
2118
|
+
for (const entry of entries || []) {
|
|
2119
|
+
if (Array.isArray(entry)) {
|
|
2120
|
+
if (entry[0] !== 'attention') {
|
|
2121
|
+
continue;
|
|
2122
|
+
}
|
|
2123
|
+
replaceKernelEntry(
|
|
2124
|
+
entry[1],
|
|
2125
|
+
derivePrefillAttentionEntry(result.kernels[entry[1]])
|
|
2126
|
+
);
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
2129
|
+
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2130
|
+
replacePrefillAttentionEntries(entry.steps);
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
replacePrefillAttentionEntries(result.prefill);
|
|
2135
|
+
|
|
2136
|
+
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2137
|
+
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2138
|
+
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2139
|
+
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2140
|
+
replaceKernelEntry(
|
|
2141
|
+
finalNormKey,
|
|
2142
|
+
deriveKernelEntryWithPrecision(
|
|
2143
|
+
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2144
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2145
|
+
)
|
|
2146
|
+
);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2150
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2151
|
+
replaceKernelEntry(
|
|
2152
|
+
lmHeadKey,
|
|
2153
|
+
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2154
|
+
);
|
|
2155
|
+
|
|
2156
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2157
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2158
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2159
|
+
if (
|
|
2160
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2161
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2162
|
+
) {
|
|
2163
|
+
replaceKernelEntry(
|
|
2164
|
+
lmHeadPrefillKey,
|
|
2165
|
+
deriveKernelEntryWithPrecision(
|
|
2166
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2167
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2168
|
+
)
|
|
2169
|
+
);
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
if (stableTextBoundary) {
|
|
2173
|
+
changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2176
|
+
return changed ? result : null;
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
export function useGemma4TextF16Activations(graph, ctx) {
|
|
2180
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
export function useGemma412BTextF16Activations(graph, ctx) {
|
|
2184
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
|
|
2185
|
+
}
|
|
2186
|
+
|
|
2187
|
+
export function useGemma431BTextF16Activations(graph, ctx) {
|
|
2188
|
+
return useGemma4TextF16Activations(graph, ctx);
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
// =============================================================================
|
|
2192
|
+
// Transform: useGemma4Int4PleAf16Activations
|
|
2193
|
+
// =============================================================================
|
|
2194
|
+
|
|
2195
|
+
/*
|
|
2196
|
+
* Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
|
|
2197
|
+
* weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
|
|
2198
|
+
* useGemma4TextF16Activations: same Q4 weight pack, kernels narrowed to f16
|
|
2199
|
+
* activations, prefill projections promoted from widetile to widetile_f16a,
|
|
2200
|
+
* decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
|
|
2201
|
+
* counterparts. Apple Metal stays disabled at the capability layer because the
|
|
2202
|
+
* fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
|
|
2203
|
+
*
|
|
2204
|
+
*/
|
|
2205
|
+
export function useGemma4Int4PleAf16Activations(graph, ctx) {
|
|
2206
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2207
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
2208
|
+
let changed = narrowed != null;
|
|
2209
|
+
|
|
2210
|
+
const replaceKernelEntry = (key, entry) => {
|
|
2211
|
+
if (!key || !entry) {
|
|
2212
|
+
return;
|
|
2213
|
+
}
|
|
2214
|
+
result.kernels[key] = entry;
|
|
2215
|
+
changed = true;
|
|
2216
|
+
};
|
|
2217
|
+
|
|
2218
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2219
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
2220
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2221
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2222
|
+
replaceKernelEntry(
|
|
2223
|
+
embedKey,
|
|
2224
|
+
deriveKernelEntryWithPrecision(
|
|
2225
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2226
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2227
|
+
)
|
|
2228
|
+
);
|
|
2229
|
+
}
|
|
2230
|
+
|
|
2231
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2232
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2233
|
+
replaceKernelEntry(
|
|
2234
|
+
decodeProjectionKey,
|
|
2235
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2236
|
+
);
|
|
2237
|
+
|
|
2238
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2239
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2240
|
+
replaceKernelEntry(
|
|
2241
|
+
prefillProjectionKey,
|
|
2242
|
+
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2243
|
+
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
2244
|
+
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2245
|
+
);
|
|
2246
|
+
|
|
2247
|
+
const replacePrefillAttentionEntries = (entries) => {
|
|
2248
|
+
for (const entry of entries || []) {
|
|
2249
|
+
if (Array.isArray(entry)) {
|
|
2250
|
+
if (entry[0] !== 'attention') {
|
|
2251
|
+
continue;
|
|
2252
|
+
}
|
|
2253
|
+
replaceKernelEntry(
|
|
2254
|
+
entry[1],
|
|
2255
|
+
deriveF16AttentionKernelEntry(result.kernels[entry[1]])
|
|
2256
|
+
);
|
|
2257
|
+
continue;
|
|
2258
|
+
}
|
|
2259
|
+
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2260
|
+
replacePrefillAttentionEntries(entry.steps);
|
|
2261
|
+
}
|
|
2262
|
+
}
|
|
2263
|
+
};
|
|
2264
|
+
replacePrefillAttentionEntries(result.prefill);
|
|
2265
|
+
|
|
2266
|
+
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2267
|
+
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2268
|
+
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2269
|
+
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2270
|
+
replaceKernelEntry(
|
|
2271
|
+
finalNormKey,
|
|
2272
|
+
deriveKernelEntryWithPrecision(
|
|
2273
|
+
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2274
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2275
|
+
)
|
|
2276
|
+
);
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2280
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2281
|
+
replaceKernelEntry(
|
|
2282
|
+
lmHeadKey,
|
|
2283
|
+
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2284
|
+
);
|
|
2285
|
+
|
|
2286
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2287
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2288
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2289
|
+
if (
|
|
2290
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2291
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2292
|
+
) {
|
|
2293
|
+
replaceKernelEntry(
|
|
2294
|
+
lmHeadPrefillKey,
|
|
2295
|
+
deriveKernelEntryWithPrecision(
|
|
2296
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2297
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2298
|
+
)
|
|
2299
|
+
);
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
return changed ? result : null;
|
|
2303
|
+
}
|
|
2304
|
+
|
|
1675
2305
|
// =============================================================================
|
|
1676
2306
|
// Composition
|
|
1677
2307
|
// =============================================================================
|
|
1678
2308
|
|
|
1679
|
-
|
|
2309
|
+
/*
|
|
1680
2310
|
* Compose multiple transforms into a single transform function.
|
|
1681
2311
|
*
|
|
1682
2312
|
* Each transform is applied sequentially. If a transform returns null
|
|
1683
2313
|
* (not applicable), the graph passes through unchanged.
|
|
1684
2314
|
*
|
|
1685
|
-
* @param {Array<(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph | null>} transforms
|
|
1686
|
-
* @returns {(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph}
|
|
1687
2315
|
*/
|
|
1688
2316
|
export function composeTransforms(...transforms) {
|
|
1689
2317
|
return (graph, ctx) => {
|
|
@@ -1698,17 +2326,42 @@ export function composeTransforms(...transforms) {
|
|
|
1698
2326
|
};
|
|
1699
2327
|
}
|
|
1700
2328
|
|
|
2329
|
+
// Session-only capability transform marker. The execution-v1 compiler applies
|
|
2330
|
+
// the matching runtime session patch; the graph itself is intentionally stable.
|
|
2331
|
+
export function disableRetainQ4KMaterialization() {
|
|
2332
|
+
return null;
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
/*
|
|
2336
|
+
* Fail-closed sentinel transform. A capability rule installs this when the
|
|
2337
|
+
* matched (modelId, runtime profile) combination is contradictory — for
|
|
2338
|
+
* example, an af32 manifest variant paired with a runtime profile that
|
|
2339
|
+
* demands f16 activations. The matcher reaches this transform only when the
|
|
2340
|
+
* earlier manifest-binding gate has been bypassed; throwing here keeps the
|
|
2341
|
+
* lane-confusion door shut at the capability layer too.
|
|
2342
|
+
*
|
|
2343
|
+
*/
|
|
2344
|
+
export function failClosedLaneMismatch(_graph, ctx) {
|
|
2345
|
+
const modelId = ctx?.modelId ?? 'unknown';
|
|
2346
|
+
const activationDtype = ctx?.activationDtype ?? 'unknown';
|
|
2347
|
+
throw new Error(
|
|
2348
|
+
`Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
|
|
2349
|
+
'The manifest variant tag is the lane identity — load the manifest variant ' +
|
|
2350
|
+
'whose compute lane matches the runtime profile.'
|
|
2351
|
+
);
|
|
2352
|
+
}
|
|
2353
|
+
|
|
1701
2354
|
// =============================================================================
|
|
1702
2355
|
// Registry
|
|
1703
2356
|
// =============================================================================
|
|
1704
2357
|
|
|
1705
|
-
/** @type {Readonly<Record<string, Function>>} */
|
|
1706
2358
|
export const TRANSFORMS = Object.freeze({
|
|
1707
2359
|
narrowToF16Activations,
|
|
1708
2360
|
removeSubgroups,
|
|
1709
2361
|
widenToF32Activations,
|
|
1710
2362
|
widenToF32CorrectnessFallback,
|
|
1711
2363
|
swapPrefillAttention,
|
|
2364
|
+
useHead256SmallPrefillAttention,
|
|
1712
2365
|
useHead256PrefillAttention,
|
|
1713
2366
|
widenProjectionWeightsToF32,
|
|
1714
2367
|
remapDenseQ4KPrefillToQ4Native,
|
|
@@ -1718,7 +2371,15 @@ export const TRANSFORMS = Object.freeze({
|
|
|
1718
2371
|
remapQ4KDecodeAttentionToGemv,
|
|
1719
2372
|
remapQ4KDecodeAttentionToFusedQ4KGemv,
|
|
1720
2373
|
remapQ4KDecodeFFNToGemv,
|
|
2374
|
+
disableRetainQ4KMaterialization,
|
|
1721
2375
|
useQwenF16PrimaryMatmuls,
|
|
2376
|
+
useQwen36F16Activations,
|
|
1722
2377
|
useQwenDecodeF16Matmuls,
|
|
2378
|
+
useGemma4Int4PleSelectiveF16Decode,
|
|
2379
|
+
useGemma4TextF16Activations,
|
|
2380
|
+
useGemma412BTextF16Activations,
|
|
2381
|
+
useGemma431BTextF16Activations,
|
|
2382
|
+
useGemma4Int4PleAf16Activations,
|
|
2383
|
+
failClosedLaneMismatch,
|
|
1723
2384
|
composeTransforms,
|
|
1724
2385
|
});
|