doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -13,16 +13,14 @@
|
|
|
13
13
|
// Helpers
|
|
14
14
|
// =============================================================================
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
/*
|
|
17
17
|
* Deep-clone an execution graph.
|
|
18
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
19
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph}
|
|
20
18
|
*/
|
|
21
19
|
function cloneGraph(graph) {
|
|
22
20
|
return structuredClone(graph);
|
|
23
21
|
}
|
|
24
22
|
|
|
25
|
-
|
|
23
|
+
/*
|
|
26
24
|
* Shader files that require subgroups even though "subgroup" is not in the filename.
|
|
27
25
|
* Online attention kernels use subgroup reductions internally.
|
|
28
26
|
*/
|
|
@@ -51,10 +49,8 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
|
51
49
|
['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
52
50
|
]);
|
|
53
51
|
|
|
54
|
-
|
|
52
|
+
/*
|
|
55
53
|
* Check whether a kernel entry requires subgroup support.
|
|
56
|
-
* @param {{ kernel: string }} kernelEntry
|
|
57
|
-
* @returns {boolean}
|
|
58
54
|
*/
|
|
59
55
|
function isSubgroupKernel(kernelEntry) {
|
|
60
56
|
if (typeof kernelEntry.kernel !== 'string') return false;
|
|
@@ -66,11 +62,8 @@ function requiresNoSubgroupFallback(kernelEntry) {
|
|
|
66
62
|
return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
|
|
67
63
|
}
|
|
68
64
|
|
|
69
|
-
|
|
65
|
+
/*
|
|
70
66
|
* Find all kernel keys in the graph whose `kernel` file matches the given filename.
|
|
71
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
72
|
-
* @param {string} filename
|
|
73
|
-
* @returns {string[]}
|
|
74
67
|
*/
|
|
75
68
|
function findKernelKeysByFile(graph, filename) {
|
|
76
69
|
const keys = [];
|
|
@@ -82,23 +75,15 @@ function findKernelKeysByFile(graph, filename) {
|
|
|
82
75
|
return keys;
|
|
83
76
|
}
|
|
84
77
|
|
|
85
|
-
|
|
78
|
+
/*
|
|
86
79
|
* Check whether any kernel in the graph uses the given shader file.
|
|
87
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
88
|
-
* @param {string} filename
|
|
89
|
-
* @returns {boolean}
|
|
90
80
|
*/
|
|
91
81
|
function hasKernelFile(graph, filename) {
|
|
92
82
|
return findKernelKeysByFile(graph, filename).length > 0;
|
|
93
83
|
}
|
|
94
84
|
|
|
95
|
-
|
|
85
|
+
/*
|
|
96
86
|
* Create a new kernel entry with the digest cleared (shader changed).
|
|
97
|
-
* @param {object} base - original kernel entry
|
|
98
|
-
* @param {string} newFile - new shader filename
|
|
99
|
-
* @param {string} newEntry - new entry point name
|
|
100
|
-
* @param {object|null} [constants] - override constants (null to remove)
|
|
101
|
-
* @returns {object}
|
|
102
87
|
*/
|
|
103
88
|
function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
104
89
|
const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
|
|
@@ -145,12 +130,8 @@ function deriveKernelPrecision(base, newFile) {
|
|
|
145
130
|
return precision;
|
|
146
131
|
}
|
|
147
132
|
|
|
148
|
-
|
|
133
|
+
/*
|
|
149
134
|
* Derive a non-colliding kernel key name.
|
|
150
|
-
* @param {object} kernels - existing kernels dict
|
|
151
|
-
* @param {string} baseKey - original key
|
|
152
|
-
* @param {string} suffix - suffix to append
|
|
153
|
-
* @returns {string}
|
|
154
135
|
*/
|
|
155
136
|
function deriveKernelKey(kernels, baseKey, suffix) {
|
|
156
137
|
const candidate = `${baseKey}${suffix}`;
|
|
@@ -164,11 +145,8 @@ function deriveKernelKey(kernels, baseKey, suffix) {
|
|
|
164
145
|
return `${candidate}_${counter}`;
|
|
165
146
|
}
|
|
166
147
|
|
|
167
|
-
|
|
148
|
+
/*
|
|
168
149
|
* Replace kernel key references in step tuples.
|
|
169
|
-
* @param {Array<Array>} steps
|
|
170
|
-
* @param {Map<string, string>} keyMap - oldKey → newKey
|
|
171
|
-
* @returns {Array<Array>}
|
|
172
150
|
*/
|
|
173
151
|
function remapStepKeys(steps, keyMap) {
|
|
174
152
|
return steps.map((step) => {
|
|
@@ -183,12 +161,8 @@ function remapStepKeys(steps, keyMap) {
|
|
|
183
161
|
});
|
|
184
162
|
}
|
|
185
163
|
|
|
186
|
-
|
|
164
|
+
/*
|
|
187
165
|
* Check whether a step tuple's kernel key resolves to the given shader file.
|
|
188
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
189
|
-
* @param {Array} step
|
|
190
|
-
* @param {string} filename
|
|
191
|
-
* @returns {boolean}
|
|
192
166
|
*/
|
|
193
167
|
function stepUsesFile(graph, step, filename) {
|
|
194
168
|
const kernelKey = step[1];
|
|
@@ -196,14 +170,9 @@ function stepUsesFile(graph, step, filename) {
|
|
|
196
170
|
return entry != null && entry.kernel === filename;
|
|
197
171
|
}
|
|
198
172
|
|
|
199
|
-
|
|
173
|
+
/*
|
|
200
174
|
* Find the first kernel key used by matching ops in a phase whose shader file
|
|
201
175
|
* satisfies the provided predicate.
|
|
202
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
203
|
-
* @param {Array<Array>} steps
|
|
204
|
-
* @param {ReadonlySet<string>} ops
|
|
205
|
-
* @param {(entry: { kernel: string, entry: string }) => boolean} predicate
|
|
206
|
-
* @returns {string | null}
|
|
207
176
|
*/
|
|
208
177
|
function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
209
178
|
for (const step of steps || []) {
|
|
@@ -218,12 +187,8 @@ function findPhaseKernelKey(graph, steps, ops, predicate) {
|
|
|
218
187
|
return null;
|
|
219
188
|
}
|
|
220
189
|
|
|
221
|
-
|
|
190
|
+
/*
|
|
222
191
|
* Find an existing kernel key by shader file and entry point.
|
|
223
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
224
|
-
* @param {string} filename
|
|
225
|
-
* @param {string} entryPoint
|
|
226
|
-
* @returns {string | null}
|
|
227
192
|
*/
|
|
228
193
|
function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
|
|
229
194
|
for (const [key, entry] of Object.entries(graph.kernels)) {
|
|
@@ -544,10 +509,25 @@ function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
|
544
509
|
return null;
|
|
545
510
|
}
|
|
546
511
|
|
|
547
|
-
function deriveQ4PrefillF32ActivationKernelEntry(base) {
|
|
512
|
+
function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
|
|
548
513
|
if (typeof base?.kernel !== 'string') {
|
|
549
514
|
return null;
|
|
550
515
|
}
|
|
516
|
+
const replacement = options.fullF32 === true
|
|
517
|
+
? {
|
|
518
|
+
kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
|
|
519
|
+
entry: 'main',
|
|
520
|
+
}
|
|
521
|
+
: {
|
|
522
|
+
kernel: 'fused_matmul_q4_widetile.wgsl',
|
|
523
|
+
entry: 'main',
|
|
524
|
+
};
|
|
525
|
+
if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
|
|
526
|
+
return deriveKernelEntryWithPrecision(
|
|
527
|
+
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
528
|
+
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
529
|
+
);
|
|
530
|
+
}
|
|
551
531
|
if (
|
|
552
532
|
base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
|
|
553
533
|
&& base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
|
|
@@ -557,7 +537,7 @@ function deriveQ4PrefillF32ActivationKernelEntry(base) {
|
|
|
557
537
|
}
|
|
558
538
|
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
559
539
|
return deriveKernelEntryWithPrecision(
|
|
560
|
-
deriveKernelEntry(base,
|
|
540
|
+
deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
|
|
561
541
|
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
562
542
|
);
|
|
563
543
|
}
|
|
@@ -589,15 +569,12 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
|
|
|
589
569
|
// Transform: removeSubgroups
|
|
590
570
|
// =============================================================================
|
|
591
571
|
|
|
592
|
-
|
|
572
|
+
/*
|
|
593
573
|
* Remove subgroup shader dependencies from decode and postLayer steps.
|
|
594
574
|
* Prefill steps are left untouched (they already use tiled matmul).
|
|
595
575
|
*
|
|
596
576
|
* Returns null if the graph has no subgroup kernels.
|
|
597
577
|
*
|
|
598
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
599
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
600
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
601
578
|
*/
|
|
602
579
|
export function removeSubgroups(graph, ctx) {
|
|
603
580
|
const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
|
|
@@ -684,11 +661,10 @@ export function removeSubgroups(graph, ctx) {
|
|
|
684
661
|
// Transform: widenToF32Activations
|
|
685
662
|
// =============================================================================
|
|
686
663
|
|
|
687
|
-
|
|
664
|
+
/*
|
|
688
665
|
* Activation-only widening: f16-activation shaders → f32-activation variants
|
|
689
666
|
* that still use f16 for weights and KV cache. Requires shader-f16 for weight
|
|
690
667
|
* and KV buffer reads.
|
|
691
|
-
* @type {ReadonlyMap<string, string>}
|
|
692
668
|
*/
|
|
693
669
|
const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
694
670
|
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
@@ -710,14 +686,13 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
710
686
|
['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
|
|
711
687
|
]);
|
|
712
688
|
|
|
713
|
-
|
|
689
|
+
/*
|
|
714
690
|
* Activation-only narrowing: f32-activation shaders that still consume f16
|
|
715
691
|
* weights/KV are rewritten onto the matching f16-activation lane.
|
|
716
692
|
*
|
|
717
693
|
* This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
|
|
718
694
|
* runtime session explicitly requests f16 activations for an execution-v1
|
|
719
695
|
* graph that was authored with conservative f32 activation defaults.
|
|
720
|
-
* @type {ReadonlyMap<string, string>}
|
|
721
696
|
*/
|
|
722
697
|
const F32_TO_F16_ACTIVATION_MAP = new Map(
|
|
723
698
|
Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
|
|
@@ -738,11 +713,10 @@ function hasExplicitF32ActivationContract(entry) {
|
|
|
738
713
|
|| precision.outputDtype === 'f32';
|
|
739
714
|
}
|
|
740
715
|
|
|
741
|
-
|
|
716
|
+
/*
|
|
742
717
|
* Correctness fallback: preserve f16 weights where possible, but widen both
|
|
743
718
|
* activations and KV-cache interactions onto the stable f32 execution lane.
|
|
744
719
|
* Used for alternate-plan recovery after finiteness failure.
|
|
745
|
-
* @type {ReadonlyMap<string, string>}
|
|
746
720
|
*/
|
|
747
721
|
const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
748
722
|
['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
|
|
@@ -767,11 +741,10 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
|
767
741
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
768
742
|
]);
|
|
769
743
|
|
|
770
|
-
|
|
744
|
+
/*
|
|
771
745
|
* Full f32 widening: every shader that uses `enable f16;` is replaced with a
|
|
772
746
|
* pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
|
|
773
747
|
* Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
|
|
774
|
-
* @type {ReadonlyMap<string, string>}
|
|
775
748
|
*/
|
|
776
749
|
const FULL_F32_SHADER_MAP = new Map([
|
|
777
750
|
// f16-activation utility kernels → f32
|
|
@@ -805,7 +778,7 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
805
778
|
['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
|
|
806
779
|
]);
|
|
807
780
|
|
|
808
|
-
|
|
781
|
+
/*
|
|
809
782
|
* Widen all f16-activation shaders to f32-activation equivalents.
|
|
810
783
|
*
|
|
811
784
|
* Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
|
|
@@ -814,9 +787,6 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
814
787
|
* NOTE: The caller is responsible for also updating session.activationDtype
|
|
815
788
|
* to reflect the widened dtype.
|
|
816
789
|
*
|
|
817
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
818
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
819
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
820
790
|
*/
|
|
821
791
|
export function widenToF32Activations(graph, ctx) {
|
|
822
792
|
// Bail out if fused f16 FFN is present — no direct f32 equivalent
|
|
@@ -830,9 +800,12 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
830
800
|
const shaderMap = ctx.capabilities?.hasF16 === false
|
|
831
801
|
? FULL_F32_SHADER_MAP
|
|
832
802
|
: F16_TO_F32_ACTIVATION_MAP;
|
|
803
|
+
const fullF32 = ctx.capabilities?.hasF16 === false;
|
|
833
804
|
|
|
834
805
|
const hasTargetShader = Object.values(graph.kernels).some(
|
|
835
806
|
(entry) => shaderMap.has(entry.kernel)
|
|
807
|
+
|| deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
808
|
+
|| deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
|
|
836
809
|
);
|
|
837
810
|
if (!hasTargetShader) {
|
|
838
811
|
return null;
|
|
@@ -842,7 +815,7 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
842
815
|
|
|
843
816
|
for (const [key, entry] of Object.entries(result.kernels)) {
|
|
844
817
|
const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
|
|
845
|
-
?? deriveQ4PrefillF32ActivationKernelEntry(entry);
|
|
818
|
+
?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
|
|
846
819
|
if (q4FallbackEntry) {
|
|
847
820
|
result.kernels[key] = q4FallbackEntry;
|
|
848
821
|
continue;
|
|
@@ -856,13 +829,10 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
856
829
|
return result;
|
|
857
830
|
}
|
|
858
831
|
|
|
859
|
-
|
|
832
|
+
/*
|
|
860
833
|
* Widen an f16 execution graph onto the stable f32 correctness lane used for
|
|
861
834
|
* alternate-plan recovery after finiteness failure.
|
|
862
835
|
*
|
|
863
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
864
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
865
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
866
836
|
*/
|
|
867
837
|
export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
868
838
|
if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
|
|
@@ -892,16 +862,13 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
|
892
862
|
return result;
|
|
893
863
|
}
|
|
894
864
|
|
|
895
|
-
|
|
865
|
+
/*
|
|
896
866
|
* Narrow f32-activation shaders back onto their f16-activation equivalents.
|
|
897
867
|
*
|
|
898
868
|
* Returns null if the graph has no supported f32-activation kernels to swap or
|
|
899
869
|
* if the runtime did not explicitly request f16 activations on an f16-capable
|
|
900
870
|
* GPU.
|
|
901
871
|
*
|
|
902
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
903
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
904
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
905
872
|
*/
|
|
906
873
|
export function narrowToF16Activations(graph, ctx) {
|
|
907
874
|
if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
@@ -932,7 +899,6 @@ export function narrowToF16Activations(graph, ctx) {
|
|
|
932
899
|
// Transform: swapPrefillAttention
|
|
933
900
|
// =============================================================================
|
|
934
901
|
|
|
935
|
-
/** @type {ReadonlyMap<string, string>} */
|
|
936
902
|
const PREFILL_ATTENTION_PAIRS = new Map([
|
|
937
903
|
['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
|
|
938
904
|
['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
@@ -960,7 +926,7 @@ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
|
|
|
960
926
|
return false;
|
|
961
927
|
}
|
|
962
928
|
|
|
963
|
-
|
|
929
|
+
/*
|
|
964
930
|
* Swap prefill attention kernel between streaming and small variants.
|
|
965
931
|
*
|
|
966
932
|
* The `opts` parameter specifies the direction:
|
|
@@ -969,10 +935,6 @@ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
|
|
|
969
935
|
* If `from`/`to` are not provided, uses the bidirectional pair map.
|
|
970
936
|
* Returns null if no matching prefill attention kernel is found.
|
|
971
937
|
*
|
|
972
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
973
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
974
|
-
* @param {{ from?: string, to?: string }} [opts]
|
|
975
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
976
938
|
*/
|
|
977
939
|
export function swapPrefillAttention(graph, ctx, opts) {
|
|
978
940
|
const from = opts?.from;
|
|
@@ -1010,12 +972,9 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
1010
972
|
// Transform: useHead256PrefillAttention
|
|
1011
973
|
// =============================================================================
|
|
1012
974
|
|
|
1013
|
-
|
|
975
|
+
/*
|
|
1014
976
|
* Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
|
|
1015
977
|
*
|
|
1016
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1017
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1018
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1019
978
|
*/
|
|
1020
979
|
export function useHead256SmallPrefillAttention(graph, ctx) {
|
|
1021
980
|
return swapPrefillAttention(graph, ctx, {
|
|
@@ -1024,12 +983,9 @@ export function useHead256SmallPrefillAttention(graph, ctx) {
|
|
|
1024
983
|
});
|
|
1025
984
|
}
|
|
1026
985
|
|
|
1027
|
-
|
|
986
|
+
/*
|
|
1028
987
|
* Promote prefill attention onto the fixed 256-dim shared-block kernel.
|
|
1029
988
|
*
|
|
1030
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1031
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1032
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1033
989
|
*/
|
|
1034
990
|
export function useHead256PrefillAttention(graph, ctx) {
|
|
1035
991
|
let current = graph;
|
|
@@ -1057,7 +1013,6 @@ export function useHead256PrefillAttention(graph, ctx) {
|
|
|
1057
1013
|
// Transform: widenProjectionWeightsToF32
|
|
1058
1014
|
// =============================================================================
|
|
1059
1015
|
|
|
1060
|
-
/** @type {ReadonlySet<string>} */
|
|
1061
1016
|
const PROJECTION_MATMUL_FILES = new Set([
|
|
1062
1017
|
'matmul_gemv_subgroup.wgsl',
|
|
1063
1018
|
'matmul_gemv_subgroup_f16a.wgsl',
|
|
@@ -1067,10 +1022,9 @@ const PROJECTION_MATMUL_FILES = new Set([
|
|
|
1067
1022
|
'matmul_f16_tiled.wgsl',
|
|
1068
1023
|
]);
|
|
1069
1024
|
|
|
1070
|
-
|
|
1025
|
+
/*
|
|
1071
1026
|
* Known layer projection ops. Only these are widened; lm_head and embed are
|
|
1072
1027
|
* excluded.
|
|
1073
|
-
* @type {ReadonlySet<string>}
|
|
1074
1028
|
*/
|
|
1075
1029
|
const LAYER_PROJECTION_OPS = new Set([
|
|
1076
1030
|
'q_proj', 'k_proj', 'v_proj', 'o_proj',
|
|
@@ -1090,7 +1044,7 @@ function resolveDensePrefillProjectionKernel(ctx) {
|
|
|
1090
1044
|
: 'matmul_f16w_f32a.wgsl';
|
|
1091
1045
|
}
|
|
1092
1046
|
|
|
1093
|
-
|
|
1047
|
+
/*
|
|
1094
1048
|
* Replace projection matmul kernels with f32 weight variants.
|
|
1095
1049
|
*
|
|
1096
1050
|
* Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
|
|
@@ -1098,9 +1052,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
|
|
|
1098
1052
|
*
|
|
1099
1053
|
* Returns null if no applicable projection kernels are found.
|
|
1100
1054
|
*
|
|
1101
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1102
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1103
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1104
1055
|
*/
|
|
1105
1056
|
export function widenProjectionWeightsToF32(graph, ctx) {
|
|
1106
1057
|
// Collect kernel keys used by layer projection steps across all phases
|
|
@@ -1152,7 +1103,7 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
1152
1103
|
// Transform: remapDenseQ4KPrefillToQ4Native
|
|
1153
1104
|
// =============================================================================
|
|
1154
1105
|
|
|
1155
|
-
|
|
1106
|
+
/*
|
|
1156
1107
|
* Replace dense prefill projection kernels with Q4-native prefill variants.
|
|
1157
1108
|
*
|
|
1158
1109
|
* This applies only when the graph already exposes a compatible fused Q4 decode
|
|
@@ -1163,9 +1114,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
|
|
|
1163
1114
|
* Returns null when the graph does not have the required dense-prefill + Q4
|
|
1164
1115
|
* decode shape.
|
|
1165
1116
|
*
|
|
1166
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1167
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1168
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1169
1117
|
*/
|
|
1170
1118
|
export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
1171
1119
|
const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
@@ -1234,7 +1182,7 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
|
1234
1182
|
// Transform: remapQ4KPrefillToDense
|
|
1235
1183
|
// =============================================================================
|
|
1236
1184
|
|
|
1237
|
-
|
|
1185
|
+
/*
|
|
1238
1186
|
* Replace fused Q4K prefill projection kernels with dense tiled variants.
|
|
1239
1187
|
*
|
|
1240
1188
|
* Decode remains unchanged so the runtime can keep using fused Q4K decode while
|
|
@@ -1242,9 +1190,6 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
|
|
|
1242
1190
|
*
|
|
1243
1191
|
* Returns null when the graph has no fused Q4K prefill projection kernels.
|
|
1244
1192
|
*
|
|
1245
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1246
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1247
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1248
1193
|
*/
|
|
1249
1194
|
export function remapQ4KPrefillToDense(graph, ctx) {
|
|
1250
1195
|
const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
|
|
@@ -1298,7 +1243,7 @@ export function remapQ4KPrefillToDense(graph, ctx) {
|
|
|
1298
1243
|
// Transform: useLinearDecodeProjectionF16
|
|
1299
1244
|
// =============================================================================
|
|
1300
1245
|
|
|
1301
|
-
|
|
1246
|
+
/*
|
|
1302
1247
|
* Remap the linear-attention q_proj decode step onto the f16-activation fused
|
|
1303
1248
|
* Q4 kernel for linear-attention layers only. Full-attention layers keep the
|
|
1304
1249
|
* manifest-wide f32 activation contract.
|
|
@@ -1310,9 +1255,6 @@ export function remapQ4KPrefillToDense(graph, ctx) {
|
|
|
1310
1255
|
* repetitive output under greedy decode). q_proj f16 is safe because the
|
|
1311
1256
|
* linear attention core absorbs the f16 input into its f32 internal state.
|
|
1312
1257
|
*
|
|
1313
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1314
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1315
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1316
1258
|
*/
|
|
1317
1259
|
export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
1318
1260
|
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
@@ -1368,7 +1310,7 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
|
1368
1310
|
// Transform: remapQ4KDecodeToGemv
|
|
1369
1311
|
// =============================================================================
|
|
1370
1312
|
|
|
1371
|
-
|
|
1313
|
+
/*
|
|
1372
1314
|
* Replace fused Q4K decode projection kernels with GEMV subgroup variants.
|
|
1373
1315
|
*
|
|
1374
1316
|
* When Q4K weights have f16 materializations (mixed/dense loader mode), the
|
|
@@ -1382,9 +1324,6 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
|
|
|
1382
1324
|
* Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
|
|
1383
1325
|
* attention, residual, activation) are left untouched.
|
|
1384
1326
|
*
|
|
1385
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1386
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1387
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1388
1327
|
*/
|
|
1389
1328
|
export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
1390
1329
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1429,7 +1368,7 @@ export function remapQ4KDecodeToGemv(graph, ctx) {
|
|
|
1429
1368
|
|
|
1430
1369
|
const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
|
|
1431
1370
|
|
|
1432
|
-
|
|
1371
|
+
/*
|
|
1433
1372
|
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
|
|
1434
1373
|
* subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
|
|
1435
1374
|
*
|
|
@@ -1438,9 +1377,6 @@ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj'
|
|
|
1438
1377
|
* their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
|
|
1439
1378
|
* loader remains in mixed-materialization mode.
|
|
1440
1379
|
*
|
|
1441
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1442
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1443
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1444
1380
|
*/
|
|
1445
1381
|
export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
1446
1382
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1497,7 +1433,7 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
|
1497
1433
|
// Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
|
|
1498
1434
|
// =============================================================================
|
|
1499
1435
|
|
|
1500
|
-
|
|
1436
|
+
/*
|
|
1501
1437
|
* Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
|
|
1502
1438
|
* optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
|
|
1503
1439
|
* cooperative loading with fast nibble extraction for maximum M=1 throughput
|
|
@@ -1514,9 +1450,6 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
|
|
|
1514
1450
|
* isKernelPathFusedQ4K stays true and the weight loader remains in
|
|
1515
1451
|
* mixed-materialization mode (Q4K retained for attention, f16 for FFN).
|
|
1516
1452
|
*
|
|
1517
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1518
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1519
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1520
1453
|
*/
|
|
1521
1454
|
export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
1522
1455
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1575,7 +1508,7 @@ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
|
|
|
1575
1508
|
|
|
1576
1509
|
const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
1577
1510
|
|
|
1578
|
-
|
|
1511
|
+
/*
|
|
1579
1512
|
* Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
|
|
1580
1513
|
* variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
|
|
1581
1514
|
*
|
|
@@ -1585,9 +1518,6 @@ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
|
|
|
1585
1518
|
* fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
|
|
1586
1519
|
* remains in mixed-materialization mode.
|
|
1587
1520
|
*
|
|
1588
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1589
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1590
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1591
1521
|
*/
|
|
1592
1522
|
export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
1593
1523
|
if (ctx.activationDtype === 'f16') {
|
|
@@ -1644,7 +1574,7 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
|
1644
1574
|
// Transform: useQwenDecodeF16Matmuls
|
|
1645
1575
|
// =============================================================================
|
|
1646
1576
|
|
|
1647
|
-
|
|
1577
|
+
/*
|
|
1648
1578
|
* Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
|
|
1649
1579
|
* kernels while keeping the manifest-wide f32 activation contract intact.
|
|
1650
1580
|
*
|
|
@@ -1656,16 +1586,8 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
|
|
|
1656
1586
|
* FFN down remains on the f32-output contract so the layer residual path stays
|
|
1657
1587
|
* numerically aligned with the manifest-owned activation dtype.
|
|
1658
1588
|
*
|
|
1659
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1660
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1661
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1662
1589
|
*/
|
|
1663
1590
|
export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
1664
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1665
|
-
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
|
|
1666
|
-
return null;
|
|
1667
|
-
}
|
|
1668
|
-
|
|
1669
1591
|
const result = cloneGraph(graph);
|
|
1670
1592
|
let changed = false;
|
|
1671
1593
|
|
|
@@ -1730,7 +1652,7 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
|
1730
1652
|
// Transform: useQwenF16PrimaryMatmuls
|
|
1731
1653
|
// =============================================================================
|
|
1732
1654
|
|
|
1733
|
-
|
|
1655
|
+
/*
|
|
1734
1656
|
* Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
|
|
1735
1657
|
* lane when the runtime explicitly requests f16 activations.
|
|
1736
1658
|
*
|
|
@@ -1742,16 +1664,8 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
|
1742
1664
|
* `o_proj` writes f16 there (observed first at the first full-attention block's
|
|
1743
1665
|
* post-attention RMSNorm).
|
|
1744
1666
|
*
|
|
1745
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1746
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1747
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1748
1667
|
*/
|
|
1749
1668
|
export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
1750
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1751
|
-
if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
|
|
1752
|
-
return null;
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
1669
|
const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
|
|
1756
1670
|
if (!layerTypes || layerTypes.length === 0) {
|
|
1757
1671
|
return null;
|
|
@@ -1848,26 +1762,11 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
|
1848
1762
|
// Transform: useQwen36F16Activations
|
|
1849
1763
|
// =============================================================================
|
|
1850
1764
|
|
|
1851
|
-
|
|
1765
|
+
/*
|
|
1852
1766
|
* Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
|
|
1853
1767
|
*
|
|
1854
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1855
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1856
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1857
1768
|
*/
|
|
1858
1769
|
export function useQwen36F16Activations(graph, ctx) {
|
|
1859
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1860
|
-
if (
|
|
1861
|
-
modelId !== 'qwen-3-6-27b-q4k-eaf16'
|
|
1862
|
-
|| ctx.activationDtype !== 'f16'
|
|
1863
|
-
|| ctx.mathDtype !== 'f16'
|
|
1864
|
-
|| ctx.accumDtype !== 'f16'
|
|
1865
|
-
|| ctx.capabilities?.hasF16 !== true
|
|
1866
|
-
|| ctx.capabilities?.hasSubgroups !== true
|
|
1867
|
-
) {
|
|
1868
|
-
return null;
|
|
1869
|
-
}
|
|
1870
|
-
|
|
1871
1770
|
const narrowed = narrowToF16Activations(graph, ctx);
|
|
1872
1771
|
const result = narrowed ?? cloneGraph(graph);
|
|
1873
1772
|
let changed = narrowed != null;
|
|
@@ -1914,8 +1813,8 @@ export function useQwen36F16Activations(graph, ctx) {
|
|
|
1914
1813
|
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
1915
1814
|
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
1916
1815
|
const prefillProjectionEntry =
|
|
1917
|
-
|
|
1918
|
-
??
|
|
1816
|
+
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
1817
|
+
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
1919
1818
|
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
|
|
1920
1819
|
if (prefillProjectionEntry) {
|
|
1921
1820
|
const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
|
|
@@ -1933,7 +1832,13 @@ export function useQwen36F16Activations(graph, ctx) {
|
|
|
1933
1832
|
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
1934
1833
|
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
1935
1834
|
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
1936
|
-
|
|
1835
|
+
const lmHeadPrefillF16Entry =
|
|
1836
|
+
deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
|
|
1837
|
+
?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
|
|
1838
|
+
?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
|
|
1839
|
+
if (lmHeadPrefillF16Entry) {
|
|
1840
|
+
replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
|
|
1841
|
+
} else if (
|
|
1937
1842
|
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
1938
1843
|
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
1939
1844
|
) {
|
|
@@ -1953,27 +1858,14 @@ export function useQwen36F16Activations(graph, ctx) {
|
|
|
1953
1858
|
// Transform: useGemma4Int4PleSelectiveF16Decode
|
|
1954
1859
|
// =============================================================================
|
|
1955
1860
|
|
|
1956
|
-
|
|
1861
|
+
/*
|
|
1957
1862
|
* Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
|
|
1958
1863
|
* explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
|
|
1959
1864
|
* attention because the repository does not currently have pure-f16 head256 or
|
|
1960
1865
|
* head512 prefill kernels.
|
|
1961
1866
|
*
|
|
1962
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1963
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1964
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1965
1867
|
*/
|
|
1966
1868
|
export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
|
|
1967
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1968
|
-
if (
|
|
1969
|
-
modelId !== 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple'
|
|
1970
|
-
|| ctx.activationDtype !== 'f16'
|
|
1971
|
-
|| ctx.capabilities?.hasF16 !== true
|
|
1972
|
-
|| ctx.capabilities?.hasSubgroups !== true
|
|
1973
|
-
) {
|
|
1974
|
-
return null;
|
|
1975
|
-
}
|
|
1976
|
-
|
|
1977
1869
|
const result = cloneGraph(graph);
|
|
1978
1870
|
let changed = false;
|
|
1979
1871
|
|
|
@@ -2044,35 +1936,95 @@ export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
|
|
|
2044
1936
|
}
|
|
2045
1937
|
|
|
2046
1938
|
// =============================================================================
|
|
2047
|
-
// Transform:
|
|
1939
|
+
// Transform: useGemma4TextF16Activations
|
|
2048
1940
|
// =============================================================================
|
|
2049
1941
|
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
2062
|
-
if (
|
|
2063
|
-
![
|
|
2064
|
-
'gemma-4-31b-it-text-q4k-ehf16-af32',
|
|
2065
|
-
'gemma-4-31b-it-text-q4k-ehf16-af16',
|
|
2066
|
-
].includes(modelId)
|
|
2067
|
-
|| ctx.activationDtype !== 'f16'
|
|
2068
|
-
|| ctx.mathDtype !== 'f16'
|
|
2069
|
-
|| ctx.accumDtype !== 'f16'
|
|
2070
|
-
|| ctx.capabilities?.hasF16 !== true
|
|
2071
|
-
|| ctx.capabilities?.hasSubgroups !== true
|
|
2072
|
-
) {
|
|
2073
|
-
return null;
|
|
1942
|
+
const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
|
|
1943
|
+
'q_proj',
|
|
1944
|
+
'k_proj',
|
|
1945
|
+
'v_proj',
|
|
1946
|
+
'gate_proj',
|
|
1947
|
+
'up_proj',
|
|
1948
|
+
]);
|
|
1949
|
+
|
|
1950
|
+
function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
|
|
1951
|
+
if (!Array.isArray(entries) || !f16Key || !f32Key) {
|
|
1952
|
+
return { entries, changed: false };
|
|
2074
1953
|
}
|
|
2075
1954
|
|
|
1955
|
+
let changed = false;
|
|
1956
|
+
const remapStep = (step) => {
|
|
1957
|
+
if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
|
|
1958
|
+
return step;
|
|
1959
|
+
}
|
|
1960
|
+
const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
|
|
1961
|
+
? f32Key
|
|
1962
|
+
: f16Key;
|
|
1963
|
+
if (step[1] === targetKey) {
|
|
1964
|
+
return step;
|
|
1965
|
+
}
|
|
1966
|
+
const replacement = [...step];
|
|
1967
|
+
replacement[1] = targetKey;
|
|
1968
|
+
changed = true;
|
|
1969
|
+
return replacement;
|
|
1970
|
+
};
|
|
1971
|
+
|
|
1972
|
+
const nextEntries = entries.map((entry) => {
|
|
1973
|
+
if (Array.isArray(entry)) {
|
|
1974
|
+
return remapStep(entry);
|
|
1975
|
+
}
|
|
1976
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
1977
|
+
return entry;
|
|
1978
|
+
}
|
|
1979
|
+
return {
|
|
1980
|
+
...entry,
|
|
1981
|
+
steps: entry.steps.map((step) => remapStep(step)),
|
|
1982
|
+
};
|
|
1983
|
+
});
|
|
1984
|
+
|
|
1985
|
+
return { entries: nextEntries, changed };
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
|
|
1989
|
+
let changed = false;
|
|
1990
|
+
|
|
1991
|
+
const replaceStepWithSourceEntry = (phaseName, op, precision) => {
|
|
1992
|
+
const phase = result[phaseName];
|
|
1993
|
+
const sourcePhase = sourceGraph[phaseName];
|
|
1994
|
+
const step = findPhaseStep(phase, op);
|
|
1995
|
+
const sourceStep = findPhaseStep(sourcePhase, op);
|
|
1996
|
+
const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
|
|
1997
|
+
const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
|
|
1998
|
+
if (!step || !sourceKey || !sourceEntry) {
|
|
1999
|
+
return;
|
|
2000
|
+
}
|
|
2001
|
+
const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
|
|
2002
|
+
result.kernels[stableKey] = precision
|
|
2003
|
+
? deriveKernelEntryWithPrecision(sourceEntry, precision)
|
|
2004
|
+
: { ...sourceEntry };
|
|
2005
|
+
const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
|
|
2006
|
+
if (phaseResult.changed) {
|
|
2007
|
+
result[phaseName] = phaseResult.steps;
|
|
2008
|
+
changed = true;
|
|
2009
|
+
}
|
|
2010
|
+
};
|
|
2011
|
+
|
|
2012
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
|
|
2013
|
+
replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
|
|
2014
|
+
}
|
|
2015
|
+
replaceStepWithSourceEntry('decode', 'attention', {
|
|
2016
|
+
activationDtype: 'f32',
|
|
2017
|
+
kvDtype: 'f16',
|
|
2018
|
+
outputDtype: 'f32',
|
|
2019
|
+
});
|
|
2020
|
+
for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
|
|
2021
|
+
replaceStepWithSourceEntry('postLayer', op, null);
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
return changed;
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
|
|
2076
2028
|
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2077
2029
|
const result = narrowed ?? cloneGraph(graph);
|
|
2078
2030
|
let changed = narrowed != null;
|
|
@@ -2084,6 +2036,31 @@ export function useGemma431BTextF16Activations(graph, ctx) {
|
|
|
2084
2036
|
result.kernels[key] = entry;
|
|
2085
2037
|
changed = true;
|
|
2086
2038
|
};
|
|
2039
|
+
const stableTextBoundary = options?.stableTextBoundary === true;
|
|
2040
|
+
|
|
2041
|
+
const derivePrefillAttentionEntry = (entry) => {
|
|
2042
|
+
if (
|
|
2043
|
+
stableTextBoundary
|
|
2044
|
+
&& typeof entry?.kernel === 'string'
|
|
2045
|
+
&& entry.kernel === 'attention_small_f16.wgsl'
|
|
2046
|
+
) {
|
|
2047
|
+
return deriveKernelEntryWithPrecision(
|
|
2048
|
+
deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
|
|
2049
|
+
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2050
|
+
);
|
|
2051
|
+
}
|
|
2052
|
+
if (
|
|
2053
|
+
stableTextBoundary
|
|
2054
|
+
&& typeof entry?.kernel === 'string'
|
|
2055
|
+
&& entry.kernel.endsWith('_f16kv.wgsl')
|
|
2056
|
+
) {
|
|
2057
|
+
return deriveKernelEntryWithPrecision(
|
|
2058
|
+
entry,
|
|
2059
|
+
{ activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
|
|
2060
|
+
);
|
|
2061
|
+
}
|
|
2062
|
+
return deriveF16AttentionKernelEntry(entry);
|
|
2063
|
+
};
|
|
2087
2064
|
|
|
2088
2065
|
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2089
2066
|
const embedKey = embedStep?.[1] ?? null;
|
|
@@ -2107,12 +2084,35 @@ export function useGemma431BTextF16Activations(graph, ctx) {
|
|
|
2107
2084
|
|
|
2108
2085
|
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2109
2086
|
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2087
|
+
const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
|
|
2088
|
+
const prefillProjectionF16Entry = stableTextBoundary
|
|
2089
|
+
? (
|
|
2090
|
+
deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2091
|
+
?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2092
|
+
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2093
|
+
)
|
|
2094
|
+
: (
|
|
2095
|
+
deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
|
|
2096
|
+
?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2097
|
+
?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
|
|
2098
|
+
);
|
|
2099
|
+
if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
|
|
2100
|
+
const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
|
|
2101
|
+
result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
|
|
2102
|
+
changed = true;
|
|
2103
|
+
const remapped = remapGemma412BPrefillProjectionEntries(
|
|
2104
|
+
result.prefill,
|
|
2105
|
+
prefillProjectionF16Key,
|
|
2106
|
+
prefillProjectionKey
|
|
2107
|
+
);
|
|
2108
|
+
result.prefill = remapped.entries;
|
|
2109
|
+
changed = changed || remapped.changed;
|
|
2110
|
+
} else {
|
|
2111
|
+
replaceKernelEntry(
|
|
2112
|
+
prefillProjectionKey,
|
|
2113
|
+
prefillProjectionF16Entry
|
|
2114
|
+
);
|
|
2115
|
+
}
|
|
2116
2116
|
|
|
2117
2117
|
const replacePrefillAttentionEntries = (entries) => {
|
|
2118
2118
|
for (const entry of entries || []) {
|
|
@@ -2122,7 +2122,7 @@ export function useGemma431BTextF16Activations(graph, ctx) {
|
|
|
2122
2122
|
}
|
|
2123
2123
|
replaceKernelEntry(
|
|
2124
2124
|
entry[1],
|
|
2125
|
-
|
|
2125
|
+
derivePrefillAttentionEntry(result.kernels[entry[1]])
|
|
2126
2126
|
);
|
|
2127
2127
|
continue;
|
|
2128
2128
|
}
|
|
@@ -2169,39 +2169,40 @@ export function useGemma431BTextF16Activations(graph, ctx) {
|
|
|
2169
2169
|
);
|
|
2170
2170
|
}
|
|
2171
2171
|
|
|
2172
|
+
if (stableTextBoundary) {
|
|
2173
|
+
changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
|
|
2174
|
+
}
|
|
2175
|
+
|
|
2172
2176
|
return changed ? result : null;
|
|
2173
2177
|
}
|
|
2174
2178
|
|
|
2179
|
+
export function useGemma4TextF16Activations(graph, ctx) {
|
|
2180
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
export function useGemma412BTextF16Activations(graph, ctx) {
|
|
2184
|
+
return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
|
|
2185
|
+
}
|
|
2186
|
+
|
|
2187
|
+
export function useGemma431BTextF16Activations(graph, ctx) {
|
|
2188
|
+
return useGemma4TextF16Activations(graph, ctx);
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2175
2191
|
// =============================================================================
|
|
2176
2192
|
// Transform: useGemma4Int4PleAf16Activations
|
|
2177
2193
|
// =============================================================================
|
|
2178
2194
|
|
|
2179
|
-
|
|
2195
|
+
/*
|
|
2180
2196
|
* Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
|
|
2181
2197
|
* weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
|
|
2182
|
-
*
|
|
2198
|
+
* useGemma4TextF16Activations: same Q4 weight pack, kernels narrowed to f16
|
|
2183
2199
|
* activations, prefill projections promoted from widetile to widetile_f16a,
|
|
2184
2200
|
* decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
|
|
2185
2201
|
* counterparts. Apple Metal stays disabled at the capability layer because the
|
|
2186
2202
|
* fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
|
|
2187
2203
|
*
|
|
2188
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
2189
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
2190
|
-
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
2191
2204
|
*/
|
|
2192
2205
|
export function useGemma4Int4PleAf16Activations(graph, ctx) {
|
|
2193
|
-
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
2194
|
-
if (
|
|
2195
|
-
modelId !== 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple'
|
|
2196
|
-
|| ctx.activationDtype !== 'f16'
|
|
2197
|
-
|| ctx.mathDtype !== 'f16'
|
|
2198
|
-
|| ctx.accumDtype !== 'f16'
|
|
2199
|
-
|| ctx.capabilities?.hasF16 !== true
|
|
2200
|
-
|| ctx.capabilities?.hasSubgroups !== true
|
|
2201
|
-
) {
|
|
2202
|
-
return null;
|
|
2203
|
-
}
|
|
2204
|
-
|
|
2205
2206
|
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2206
2207
|
const result = narrowed ?? cloneGraph(graph);
|
|
2207
2208
|
let changed = narrowed != null;
|
|
@@ -2305,14 +2306,12 @@ export function useGemma4Int4PleAf16Activations(graph, ctx) {
|
|
|
2305
2306
|
// Composition
|
|
2306
2307
|
// =============================================================================
|
|
2307
2308
|
|
|
2308
|
-
|
|
2309
|
+
/*
|
|
2309
2310
|
* Compose multiple transforms into a single transform function.
|
|
2310
2311
|
*
|
|
2311
2312
|
* Each transform is applied sequentially. If a transform returns null
|
|
2312
2313
|
* (not applicable), the graph passes through unchanged.
|
|
2313
2314
|
*
|
|
2314
|
-
* @param {Array<(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph | null>} transforms
|
|
2315
|
-
* @returns {(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph}
|
|
2316
2315
|
*/
|
|
2317
2316
|
export function composeTransforms(...transforms) {
|
|
2318
2317
|
return (graph, ctx) => {
|
|
@@ -2333,7 +2332,7 @@ export function disableRetainQ4KMaterialization() {
|
|
|
2333
2332
|
return null;
|
|
2334
2333
|
}
|
|
2335
2334
|
|
|
2336
|
-
|
|
2335
|
+
/*
|
|
2337
2336
|
* Fail-closed sentinel transform. A capability rule installs this when the
|
|
2338
2337
|
* matched (modelId, runtime profile) combination is contradictory — for
|
|
2339
2338
|
* example, an af32 manifest variant paired with a runtime profile that
|
|
@@ -2341,9 +2340,6 @@ export function disableRetainQ4KMaterialization() {
|
|
|
2341
2340
|
* earlier manifest-binding gate has been bypassed; throwing here keeps the
|
|
2342
2341
|
* lane-confusion door shut at the capability layer too.
|
|
2343
2342
|
*
|
|
2344
|
-
* @param {import('./execution-graph-transforms.js').ExecutionGraph} _graph
|
|
2345
|
-
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
2346
|
-
* @returns {never}
|
|
2347
2343
|
*/
|
|
2348
2344
|
export function failClosedLaneMismatch(_graph, ctx) {
|
|
2349
2345
|
const modelId = ctx?.modelId ?? 'unknown';
|
|
@@ -2359,7 +2355,6 @@ export function failClosedLaneMismatch(_graph, ctx) {
|
|
|
2359
2355
|
// Registry
|
|
2360
2356
|
// =============================================================================
|
|
2361
2357
|
|
|
2362
|
-
/** @type {Readonly<Record<string, Function>>} */
|
|
2363
2358
|
export const TRANSFORMS = Object.freeze({
|
|
2364
2359
|
narrowToF16Activations,
|
|
2365
2360
|
removeSubgroups,
|
|
@@ -2381,6 +2376,8 @@ export const TRANSFORMS = Object.freeze({
|
|
|
2381
2376
|
useQwen36F16Activations,
|
|
2382
2377
|
useQwenDecodeF16Matmuls,
|
|
2383
2378
|
useGemma4Int4PleSelectiveF16Decode,
|
|
2379
|
+
useGemma4TextF16Activations,
|
|
2380
|
+
useGemma412BTextF16Activations,
|
|
2384
2381
|
useGemma431BTextF16Activations,
|
|
2385
2382
|
useGemma4Int4PleAf16Activations,
|
|
2386
2383
|
failClosedLaneMismatch,
|