doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -35,15 +35,20 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
|
35
35
|
['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
36
36
|
['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
37
37
|
['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
38
|
+
['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
38
39
|
['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
39
40
|
['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
41
|
+
['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
40
42
|
['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
41
43
|
['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
42
44
|
['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
45
|
+
['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
43
46
|
['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
44
47
|
['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
45
48
|
['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
46
49
|
['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
|
|
50
|
+
['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
51
|
+
['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
47
52
|
]);
|
|
48
53
|
|
|
49
54
|
/**
|
|
@@ -111,6 +116,11 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
|
|
|
111
116
|
return derived;
|
|
112
117
|
}
|
|
113
118
|
|
|
119
|
+
function deriveMappedKernelEntry(base, newFile) {
|
|
120
|
+
const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
|
|
121
|
+
return deriveKernelEntry(base, newFile, newEntry);
|
|
122
|
+
}
|
|
123
|
+
|
|
114
124
|
function deriveKernelPrecision(base, newFile) {
|
|
115
125
|
const precision = base.precision ? { ...base.precision } : {};
|
|
116
126
|
const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
|
|
@@ -292,6 +302,28 @@ function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
|
|
|
292
302
|
return { steps: nextSteps, changed };
|
|
293
303
|
}
|
|
294
304
|
|
|
305
|
+
function findPhaseStep(steps, op) {
|
|
306
|
+
if (!Array.isArray(steps) || !op) {
|
|
307
|
+
return null;
|
|
308
|
+
}
|
|
309
|
+
for (const entry of steps) {
|
|
310
|
+
if (Array.isArray(entry)) {
|
|
311
|
+
if (entry[0] === op) {
|
|
312
|
+
return entry;
|
|
313
|
+
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
const nested = findPhaseStep(entry.steps, op);
|
|
320
|
+
if (nested) {
|
|
321
|
+
return nested;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
|
|
295
327
|
function deriveKernelEntryWithPrecision(base, precision) {
|
|
296
328
|
return {
|
|
297
329
|
...base,
|
|
@@ -307,6 +339,7 @@ const ATTENTION_F16KV_TO_F16_MAP = new Map([
|
|
|
307
339
|
['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
|
|
308
340
|
['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
|
|
309
341
|
['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
|
|
342
|
+
['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
|
|
310
343
|
]);
|
|
311
344
|
|
|
312
345
|
function deriveF16AttentionKernelEntry(base) {
|
|
@@ -366,6 +399,29 @@ function deriveLmHeadDecodeF16KernelEntry(base) {
|
|
|
366
399
|
return null;
|
|
367
400
|
}
|
|
368
401
|
|
|
402
|
+
function deriveDenseDecodeF16KernelEntry(base) {
|
|
403
|
+
if (typeof base?.kernel !== 'string') {
|
|
404
|
+
return null;
|
|
405
|
+
}
|
|
406
|
+
const precision = {
|
|
407
|
+
inputDtype: 'f16',
|
|
408
|
+
outputDtype: 'f16',
|
|
409
|
+
};
|
|
410
|
+
if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
|
|
411
|
+
return {
|
|
412
|
+
...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
|
|
413
|
+
precision: {
|
|
414
|
+
...(base.precision ?? {}),
|
|
415
|
+
...precision,
|
|
416
|
+
},
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
|
|
420
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
421
|
+
}
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
|
|
369
425
|
function deriveQ4DecodeF16KernelEntry(base) {
|
|
370
426
|
if (typeof base?.kernel !== 'string') {
|
|
371
427
|
return null;
|
|
@@ -400,6 +456,9 @@ function deriveQ4PrefillF16KernelEntry(base) {
|
|
|
400
456
|
inputDtype: 'f16',
|
|
401
457
|
outputDtype: 'f16',
|
|
402
458
|
};
|
|
459
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
460
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
461
|
+
}
|
|
403
462
|
if (base.kernel.startsWith('fused_matmul_q4_batched')) {
|
|
404
463
|
return {
|
|
405
464
|
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
|
|
@@ -412,6 +471,53 @@ function deriveQ4PrefillF16KernelEntry(base) {
|
|
|
412
471
|
return null;
|
|
413
472
|
}
|
|
414
473
|
|
|
474
|
+
function deriveQ4WideTilePrefillF16KernelEntry(base) {
|
|
475
|
+
if (typeof base?.kernel !== 'string') {
|
|
476
|
+
return null;
|
|
477
|
+
}
|
|
478
|
+
const precision = {
|
|
479
|
+
inputDtype: 'f16',
|
|
480
|
+
outputDtype: 'f16',
|
|
481
|
+
};
|
|
482
|
+
if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
|
|
483
|
+
return {
|
|
484
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
|
|
485
|
+
precision: {
|
|
486
|
+
...(base.precision ?? {}),
|
|
487
|
+
...precision,
|
|
488
|
+
},
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
492
|
+
return deriveKernelEntryWithPrecision(base, precision);
|
|
493
|
+
}
|
|
494
|
+
return null;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function deriveQ4PrefillF16AccumKernelEntry(base) {
|
|
498
|
+
if (typeof base?.kernel !== 'string') {
|
|
499
|
+
return null;
|
|
500
|
+
}
|
|
501
|
+
const precision = {
|
|
502
|
+
inputDtype: 'f16',
|
|
503
|
+
outputDtype: 'f16',
|
|
504
|
+
};
|
|
505
|
+
if (
|
|
506
|
+
base.kernel === 'fused_matmul_q4_widetile.wgsl'
|
|
507
|
+
|| base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
508
|
+
|| base.kernel.startsWith('fused_matmul_q4_batched')
|
|
509
|
+
) {
|
|
510
|
+
return {
|
|
511
|
+
...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
|
|
512
|
+
precision: {
|
|
513
|
+
...(base.precision ?? {}),
|
|
514
|
+
...precision,
|
|
515
|
+
},
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
return null;
|
|
519
|
+
}
|
|
520
|
+
|
|
415
521
|
function deriveQ4DecodeF32ActivationKernelEntry(base) {
|
|
416
522
|
if (typeof base?.kernel !== 'string') {
|
|
417
523
|
return null;
|
|
@@ -445,9 +551,16 @@ function deriveQ4PrefillF32ActivationKernelEntry(base) {
|
|
|
445
551
|
if (
|
|
446
552
|
base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
|
|
447
553
|
&& base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
|
|
554
|
+
&& base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
|
|
448
555
|
) {
|
|
449
556
|
return null;
|
|
450
557
|
}
|
|
558
|
+
if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
|
|
559
|
+
return deriveKernelEntryWithPrecision(
|
|
560
|
+
deriveKernelEntry(base, 'fused_matmul_q4_widetile.wgsl', 'main', null),
|
|
561
|
+
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
562
|
+
);
|
|
563
|
+
}
|
|
451
564
|
return deriveKernelEntryWithPrecision(
|
|
452
565
|
deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
|
|
453
566
|
{ inputDtype: 'f32', outputDtype: 'f32' }
|
|
@@ -582,8 +695,11 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
582
695
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
583
696
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
584
697
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
698
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
585
699
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
586
700
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
701
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
702
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
587
703
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
588
704
|
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
589
705
|
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
@@ -591,6 +707,7 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
591
707
|
['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
|
|
592
708
|
['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
|
|
593
709
|
['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
|
|
710
|
+
['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
|
|
594
711
|
]);
|
|
595
712
|
|
|
596
713
|
/**
|
|
@@ -605,6 +722,11 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
605
722
|
const F32_TO_F16_ACTIVATION_MAP = new Map(
|
|
606
723
|
Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
|
|
607
724
|
);
|
|
725
|
+
F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
|
|
726
|
+
F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
|
|
727
|
+
// head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
|
|
728
|
+
// f16 requests fail-closed until that path has its own evidence.
|
|
729
|
+
F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
|
|
608
730
|
|
|
609
731
|
function hasExplicitF32ActivationContract(entry) {
|
|
610
732
|
const precision = entry?.precision;
|
|
@@ -627,17 +749,20 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
|
|
|
627
749
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
628
750
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
629
751
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
752
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
630
753
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
631
754
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
755
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
756
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
632
757
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
|
|
633
758
|
['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
|
|
634
759
|
['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
|
|
635
|
-
['attention_decode_online_f16.wgsl', '
|
|
636
|
-
['attention_decode_chunked_f16.wgsl', '
|
|
760
|
+
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
761
|
+
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
637
762
|
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
638
763
|
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
639
|
-
['attention_decode_online_f16kv.wgsl', '
|
|
640
|
-
['attention_decode_chunked_f16kv.wgsl', '
|
|
764
|
+
['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
765
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
641
766
|
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
642
767
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
643
768
|
]);
|
|
@@ -654,8 +779,11 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
654
779
|
['rope_f16.wgsl', 'rope.wgsl'],
|
|
655
780
|
['residual_f16.wgsl', 'residual.wgsl'],
|
|
656
781
|
['gelu_f16.wgsl', 'gelu.wgsl'],
|
|
782
|
+
['silu_f16.wgsl', 'silu.wgsl'],
|
|
657
783
|
['sample_f16.wgsl', 'sample.wgsl'],
|
|
658
784
|
['gather_f16.wgsl', 'gather.wgsl'],
|
|
785
|
+
['gather_f16_f16_out.wgsl', 'gather.wgsl'],
|
|
786
|
+
['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
|
|
659
787
|
// f16-activation matmul → f32
|
|
660
788
|
['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
|
|
661
789
|
['matmul_f16.wgsl', 'matmul_f32.wgsl'],
|
|
@@ -665,13 +793,13 @@ const FULL_F32_SHADER_MAP = new Map([
|
|
|
665
793
|
['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
|
|
666
794
|
['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
|
|
667
795
|
// f16-activation attention → f32
|
|
668
|
-
['attention_decode_online_f16.wgsl', '
|
|
669
|
-
['attention_decode_chunked_f16.wgsl', '
|
|
796
|
+
['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
|
|
797
|
+
['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
|
|
670
798
|
['attention_small_f16.wgsl', 'attention_small.wgsl'],
|
|
671
799
|
['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
|
|
672
800
|
// f16kv attention (f32 Q, f16 KV) → f32
|
|
673
|
-
['attention_decode_online_f16kv.wgsl', '
|
|
674
|
-
['attention_decode_chunked_f16kv.wgsl', '
|
|
801
|
+
['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
802
|
+
['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
675
803
|
['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
|
|
676
804
|
['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
|
|
677
805
|
['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
|
|
@@ -721,7 +849,7 @@ export function widenToF32Activations(graph, ctx) {
|
|
|
721
849
|
}
|
|
722
850
|
const replacement = shaderMap.get(entry.kernel);
|
|
723
851
|
if (replacement !== undefined) {
|
|
724
|
-
result.kernels[key] =
|
|
852
|
+
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
725
853
|
}
|
|
726
854
|
}
|
|
727
855
|
|
|
@@ -758,7 +886,7 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
|
|
|
758
886
|
}
|
|
759
887
|
const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
|
|
760
888
|
if (replacement !== undefined) {
|
|
761
|
-
result.kernels[key] =
|
|
889
|
+
result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
|
|
762
890
|
}
|
|
763
891
|
}
|
|
764
892
|
return result;
|
|
@@ -812,6 +940,26 @@ const PREFILL_ATTENTION_PAIRS = new Map([
|
|
|
812
940
|
['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
|
|
813
941
|
]);
|
|
814
942
|
|
|
943
|
+
function graphUsesKernelKeyInPrefill(graph, kernelKey) {
|
|
944
|
+
for (const entry of graph.prefill || []) {
|
|
945
|
+
if (Array.isArray(entry)) {
|
|
946
|
+
if (entry[1] === kernelKey) {
|
|
947
|
+
return true;
|
|
948
|
+
}
|
|
949
|
+
continue;
|
|
950
|
+
}
|
|
951
|
+
if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
|
|
952
|
+
continue;
|
|
953
|
+
}
|
|
954
|
+
for (const step of entry.steps) {
|
|
955
|
+
if (Array.isArray(step) && step[1] === kernelKey) {
|
|
956
|
+
return true;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
return false;
|
|
961
|
+
}
|
|
962
|
+
|
|
815
963
|
/**
|
|
816
964
|
* Swap prefill attention kernel between streaming and small variants.
|
|
817
965
|
*
|
|
@@ -847,8 +995,7 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
847
995
|
}
|
|
848
996
|
|
|
849
997
|
if (target !== undefined) {
|
|
850
|
-
|
|
851
|
-
const usedInPrefill = (graph.prefill || []).some((step) => step[1] === key);
|
|
998
|
+
const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
|
|
852
999
|
if (usedInPrefill) {
|
|
853
1000
|
result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
|
|
854
1001
|
changed = true;
|
|
@@ -863,6 +1010,20 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
863
1010
|
// Transform: useHead256PrefillAttention
|
|
864
1011
|
// =============================================================================
|
|
865
1012
|
|
|
1013
|
+
/**
|
|
1014
|
+
* Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
|
|
1015
|
+
*
|
|
1016
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1017
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1018
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1019
|
+
*/
|
|
1020
|
+
export function useHead256SmallPrefillAttention(graph, ctx) {
|
|
1021
|
+
return swapPrefillAttention(graph, ctx, {
|
|
1022
|
+
from: 'attention_small_f16kv.wgsl',
|
|
1023
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
|
|
866
1027
|
/**
|
|
867
1028
|
* Promote prefill attention onto the fixed 256-dim shared-block kernel.
|
|
868
1029
|
*
|
|
@@ -871,16 +1032,25 @@ export function swapPrefillAttention(graph, ctx, opts) {
|
|
|
871
1032
|
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
872
1033
|
*/
|
|
873
1034
|
export function useHead256PrefillAttention(graph, ctx) {
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
1035
|
+
let current = graph;
|
|
1036
|
+
let changed = false;
|
|
1037
|
+
|
|
1038
|
+
const smallRemap = useHead256SmallPrefillAttention(current, ctx);
|
|
1039
|
+
if (smallRemap) {
|
|
1040
|
+
current = smallRemap;
|
|
1041
|
+
changed = true;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
const streamingRemap = swapPrefillAttention(current, ctx, {
|
|
1045
|
+
from: 'attention_streaming_f16kv.wgsl',
|
|
1046
|
+
to: 'attention_head256_f16kv.wgsl',
|
|
1047
|
+
});
|
|
1048
|
+
if (streamingRemap) {
|
|
1049
|
+
current = streamingRemap;
|
|
1050
|
+
changed = true;
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
return changed ? current : null;
|
|
884
1054
|
}
|
|
885
1055
|
|
|
886
1056
|
// =============================================================================
|
|
@@ -1561,14 +1731,16 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
|
|
|
1561
1731
|
// =============================================================================
|
|
1562
1732
|
|
|
1563
1733
|
/**
|
|
1564
|
-
* Promote the Qwen 3.5 0.8B execution graph onto its
|
|
1565
|
-
* when the runtime explicitly requests f16 activations.
|
|
1734
|
+
* Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
|
|
1735
|
+
* lane when the runtime explicitly requests f16 activations.
|
|
1566
1736
|
*
|
|
1567
|
-
* This transform narrows the
|
|
1568
|
-
* stable manifest-owned f32-output kernel. The
|
|
1569
|
-
*
|
|
1570
|
-
*
|
|
1571
|
-
*
|
|
1737
|
+
* This transform narrows the decode projection and LM-head path while keeping
|
|
1738
|
+
* `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
|
|
1739
|
+
* already authored in the manifest on WideTile/head256 kernels and remains
|
|
1740
|
+
* manifest-owned. The residual stream feeds directly into the next RMSNorm, and
|
|
1741
|
+
* the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
|
|
1742
|
+
* `o_proj` writes f16 there (observed first at the first full-attention block's
|
|
1743
|
+
* post-attention RMSNorm).
|
|
1572
1744
|
*
|
|
1573
1745
|
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1574
1746
|
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
@@ -1672,6 +1844,463 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
|
|
|
1672
1844
|
return changed ? result : null;
|
|
1673
1845
|
}
|
|
1674
1846
|
|
|
1847
|
+
// =============================================================================
|
|
1848
|
+
// Transform: useQwen36F16Activations
|
|
1849
|
+
// =============================================================================
|
|
1850
|
+
|
|
1851
|
+
/**
|
|
1852
|
+
* Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
|
|
1853
|
+
*
|
|
1854
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1855
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1856
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1857
|
+
*/
|
|
1858
|
+
export function useQwen36F16Activations(graph, ctx) {
|
|
1859
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1860
|
+
if (
|
|
1861
|
+
modelId !== 'qwen-3-6-27b-q4k-eaf16'
|
|
1862
|
+
|| ctx.activationDtype !== 'f16'
|
|
1863
|
+
|| ctx.mathDtype !== 'f16'
|
|
1864
|
+
|| ctx.accumDtype !== 'f16'
|
|
1865
|
+
|| ctx.capabilities?.hasF16 !== true
|
|
1866
|
+
|| ctx.capabilities?.hasSubgroups !== true
|
|
1867
|
+
) {
|
|
1868
|
+
return null;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
1872
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
1873
|
+
let changed = narrowed != null;
|
|
1874
|
+
|
|
1875
|
+
const replaceKernelEntry = (key, entry) => {
|
|
1876
|
+
if (!key || !entry) {
|
|
1877
|
+
return;
|
|
1878
|
+
}
|
|
1879
|
+
result.kernels[key] = entry;
|
|
1880
|
+
changed = true;
|
|
1881
|
+
};
|
|
1882
|
+
const replaceOps = (phaseName, ops, kernelKey) => {
|
|
1883
|
+
for (const op of ops) {
|
|
1884
|
+
const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
|
|
1885
|
+
if (phaseResult.changed) {
|
|
1886
|
+
result[phaseName] = phaseResult.steps;
|
|
1887
|
+
changed = true;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
};
|
|
1891
|
+
|
|
1892
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
1893
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
1894
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
1895
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
1896
|
+
replaceKernelEntry(
|
|
1897
|
+
embedKey,
|
|
1898
|
+
deriveKernelEntryWithPrecision(
|
|
1899
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
1900
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1901
|
+
)
|
|
1902
|
+
);
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
1906
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
1907
|
+
const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
|
|
1908
|
+
if (decodeProjectionEntry) {
|
|
1909
|
+
const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
|
|
1910
|
+
result.kernels[f16Key] = decodeProjectionEntry;
|
|
1911
|
+
replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
1915
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
1916
|
+
const prefillProjectionEntry =
|
|
1917
|
+
deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
1918
|
+
?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
1919
|
+
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
|
|
1920
|
+
if (prefillProjectionEntry) {
|
|
1921
|
+
const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
|
|
1922
|
+
result.kernels[f16Key] = prefillProjectionEntry;
|
|
1923
|
+
replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
1927
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
1928
|
+
replaceKernelEntry(
|
|
1929
|
+
lmHeadKey,
|
|
1930
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
1931
|
+
);
|
|
1932
|
+
|
|
1933
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
1934
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
1935
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
1936
|
+
if (
|
|
1937
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
1938
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
1939
|
+
) {
|
|
1940
|
+
replaceKernelEntry(
|
|
1941
|
+
lmHeadPrefillKey,
|
|
1942
|
+
deriveKernelEntryWithPrecision(
|
|
1943
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
1944
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
1945
|
+
)
|
|
1946
|
+
);
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
return changed ? result : null;
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
// =============================================================================
|
|
1953
|
+
// Transform: useGemma4Int4PleSelectiveF16Decode
|
|
1954
|
+
// =============================================================================
|
|
1955
|
+
|
|
1956
|
+
/**
|
|
1957
|
+
* Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
|
|
1958
|
+
* explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
|
|
1959
|
+
* attention because the repository does not currently have pure-f16 head256 or
|
|
1960
|
+
* head512 prefill kernels.
|
|
1961
|
+
*
|
|
1962
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
1963
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
1964
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
1965
|
+
*/
|
|
1966
|
+
export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
|
|
1967
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
1968
|
+
if (
|
|
1969
|
+
modelId !== 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple'
|
|
1970
|
+
|| ctx.activationDtype !== 'f16'
|
|
1971
|
+
|| ctx.capabilities?.hasF16 !== true
|
|
1972
|
+
|| ctx.capabilities?.hasSubgroups !== true
|
|
1973
|
+
) {
|
|
1974
|
+
return null;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
const result = cloneGraph(graph);
|
|
1978
|
+
let changed = false;
|
|
1979
|
+
|
|
1980
|
+
const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
|
|
1981
|
+
const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
|
|
1982
|
+
const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
|
|
1983
|
+
if (decodeProjectionStep && decodeProjectionEntry) {
|
|
1984
|
+
const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
|
|
1985
|
+
result.kernels[decodeProjectionKey] = decodeProjectionEntry;
|
|
1986
|
+
for (const op of ['q_proj', 'k_proj', 'v_proj']) {
|
|
1987
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
|
|
1988
|
+
if (phaseResult.changed) {
|
|
1989
|
+
result.decode = phaseResult.steps;
|
|
1990
|
+
changed = true;
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
for (const op of ['rope_q', 'rope_k']) {
|
|
1996
|
+
const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
|
|
1997
|
+
const kernelKey = step?.[1];
|
|
1998
|
+
const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
|
|
1999
|
+
const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
|
|
2000
|
+
if (!step || !kernelKey || !replacement) {
|
|
2001
|
+
continue;
|
|
2002
|
+
}
|
|
2003
|
+
const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
|
|
2004
|
+
result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
|
|
2005
|
+
deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
|
|
2006
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2007
|
+
);
|
|
2008
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
|
|
2009
|
+
if (phaseResult.changed) {
|
|
2010
|
+
result.decode = phaseResult.steps;
|
|
2011
|
+
changed = true;
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
|
|
2016
|
+
const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
|
|
2017
|
+
const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
|
|
2018
|
+
if (attentionStep && attentionEntry) {
|
|
2019
|
+
const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
|
|
2020
|
+
result.kernels[attentionKey] = attentionEntry;
|
|
2021
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
|
|
2022
|
+
if (phaseResult.changed) {
|
|
2023
|
+
result.decode = phaseResult.steps;
|
|
2024
|
+
changed = true;
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
|
|
2029
|
+
const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
|
|
2030
|
+
if (oProjStep && oProjKernel) {
|
|
2031
|
+
const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
|
|
2032
|
+
result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
|
|
2033
|
+
inputDtype: 'f32',
|
|
2034
|
+
outputDtype: 'f32',
|
|
2035
|
+
});
|
|
2036
|
+
const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
|
|
2037
|
+
if (phaseResult.changed) {
|
|
2038
|
+
result.decode = phaseResult.steps;
|
|
2039
|
+
changed = true;
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
return changed ? result : null;
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
// =============================================================================
|
|
2047
|
+
// Transform: useGemma431BTextF16Activations
|
|
2048
|
+
// =============================================================================
|
|
2049
|
+
|
|
2050
|
+
/**
|
|
2051
|
+
* Promote the Gemma 4 31B text Q4K graph onto an experimental all-f16 request
|
|
2052
|
+
* lane. This keeps the same Q4K weight pack but remaps execution-v1 kernels so
|
|
2053
|
+
* Q4 projections, attention, residual/norm/activation utilities, lm_head, and
|
|
2054
|
+
* sampling consume and produce f16 activations where a matching kernel exists.
|
|
2055
|
+
*
|
|
2056
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
2057
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
2058
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
2059
|
+
*/
|
|
2060
|
+
export function useGemma431BTextF16Activations(graph, ctx) {
|
|
2061
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
2062
|
+
if (
|
|
2063
|
+
![
|
|
2064
|
+
'gemma-4-31b-it-text-q4k-ehf16-af32',
|
|
2065
|
+
'gemma-4-31b-it-text-q4k-ehf16-af16',
|
|
2066
|
+
].includes(modelId)
|
|
2067
|
+
|| ctx.activationDtype !== 'f16'
|
|
2068
|
+
|| ctx.mathDtype !== 'f16'
|
|
2069
|
+
|| ctx.accumDtype !== 'f16'
|
|
2070
|
+
|| ctx.capabilities?.hasF16 !== true
|
|
2071
|
+
|| ctx.capabilities?.hasSubgroups !== true
|
|
2072
|
+
) {
|
|
2073
|
+
return null;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2077
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
2078
|
+
let changed = narrowed != null;
|
|
2079
|
+
|
|
2080
|
+
const replaceKernelEntry = (key, entry) => {
|
|
2081
|
+
if (!key || !entry) {
|
|
2082
|
+
return;
|
|
2083
|
+
}
|
|
2084
|
+
result.kernels[key] = entry;
|
|
2085
|
+
changed = true;
|
|
2086
|
+
};
|
|
2087
|
+
|
|
2088
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2089
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
2090
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2091
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2092
|
+
replaceKernelEntry(
|
|
2093
|
+
embedKey,
|
|
2094
|
+
deriveKernelEntryWithPrecision(
|
|
2095
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2096
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2097
|
+
)
|
|
2098
|
+
);
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2102
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2103
|
+
replaceKernelEntry(
|
|
2104
|
+
decodeProjectionKey,
|
|
2105
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2106
|
+
);
|
|
2107
|
+
|
|
2108
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2109
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2110
|
+
replaceKernelEntry(
|
|
2111
|
+
prefillProjectionKey,
|
|
2112
|
+
deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
2113
|
+
?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2114
|
+
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2115
|
+
);
|
|
2116
|
+
|
|
2117
|
+
const replacePrefillAttentionEntries = (entries) => {
|
|
2118
|
+
for (const entry of entries || []) {
|
|
2119
|
+
if (Array.isArray(entry)) {
|
|
2120
|
+
if (entry[0] !== 'attention') {
|
|
2121
|
+
continue;
|
|
2122
|
+
}
|
|
2123
|
+
replaceKernelEntry(
|
|
2124
|
+
entry[1],
|
|
2125
|
+
deriveF16AttentionKernelEntry(result.kernels[entry[1]])
|
|
2126
|
+
);
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
2129
|
+
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2130
|
+
replacePrefillAttentionEntries(entry.steps);
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
replacePrefillAttentionEntries(result.prefill);
|
|
2135
|
+
|
|
2136
|
+
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2137
|
+
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2138
|
+
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2139
|
+
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2140
|
+
replaceKernelEntry(
|
|
2141
|
+
finalNormKey,
|
|
2142
|
+
deriveKernelEntryWithPrecision(
|
|
2143
|
+
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2144
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2145
|
+
)
|
|
2146
|
+
);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2150
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2151
|
+
replaceKernelEntry(
|
|
2152
|
+
lmHeadKey,
|
|
2153
|
+
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2154
|
+
);
|
|
2155
|
+
|
|
2156
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2157
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2158
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2159
|
+
if (
|
|
2160
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2161
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2162
|
+
) {
|
|
2163
|
+
replaceKernelEntry(
|
|
2164
|
+
lmHeadPrefillKey,
|
|
2165
|
+
deriveKernelEntryWithPrecision(
|
|
2166
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2167
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2168
|
+
)
|
|
2169
|
+
);
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
return changed ? result : null;
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
// =============================================================================
|
|
2176
|
+
// Transform: useGemma4Int4PleAf16Activations
|
|
2177
|
+
// =============================================================================
|
|
2178
|
+
|
|
2179
|
+
/**
|
|
2180
|
+
* Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
|
|
2181
|
+
* weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
|
|
2182
|
+
* useGemma431BTextF16Activations: same Q4 weight pack, kernels narrowed to f16
|
|
2183
|
+
* activations, prefill projections promoted from widetile to widetile_f16a,
|
|
2184
|
+
* decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
|
|
2185
|
+
* counterparts. Apple Metal stays disabled at the capability layer because the
|
|
2186
|
+
* fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
|
|
2187
|
+
*
|
|
2188
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
|
|
2189
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
2190
|
+
* @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
|
|
2191
|
+
*/
|
|
2192
|
+
export function useGemma4Int4PleAf16Activations(graph, ctx) {
|
|
2193
|
+
const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
|
|
2194
|
+
if (
|
|
2195
|
+
modelId !== 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple'
|
|
2196
|
+
|| ctx.activationDtype !== 'f16'
|
|
2197
|
+
|| ctx.mathDtype !== 'f16'
|
|
2198
|
+
|| ctx.accumDtype !== 'f16'
|
|
2199
|
+
|| ctx.capabilities?.hasF16 !== true
|
|
2200
|
+
|| ctx.capabilities?.hasSubgroups !== true
|
|
2201
|
+
) {
|
|
2202
|
+
return null;
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
const narrowed = narrowToF16Activations(graph, ctx);
|
|
2206
|
+
const result = narrowed ?? cloneGraph(graph);
|
|
2207
|
+
let changed = narrowed != null;
|
|
2208
|
+
|
|
2209
|
+
const replaceKernelEntry = (key, entry) => {
|
|
2210
|
+
if (!key || !entry) {
|
|
2211
|
+
return;
|
|
2212
|
+
}
|
|
2213
|
+
result.kernels[key] = entry;
|
|
2214
|
+
changed = true;
|
|
2215
|
+
};
|
|
2216
|
+
|
|
2217
|
+
const embedStep = findPhaseStep(result.preLayer, 'embed');
|
|
2218
|
+
const embedKey = embedStep?.[1] ?? null;
|
|
2219
|
+
const embedEntry = embedKey ? result.kernels[embedKey] : null;
|
|
2220
|
+
if (embedEntry?.kernel === 'gather_f16.wgsl') {
|
|
2221
|
+
replaceKernelEntry(
|
|
2222
|
+
embedKey,
|
|
2223
|
+
deriveKernelEntryWithPrecision(
|
|
2224
|
+
deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
|
|
2225
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2226
|
+
)
|
|
2227
|
+
);
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2230
|
+
const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
|
|
2231
|
+
const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
|
|
2232
|
+
replaceKernelEntry(
|
|
2233
|
+
decodeProjectionKey,
|
|
2234
|
+
deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
|
|
2235
|
+
);
|
|
2236
|
+
|
|
2237
|
+
const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
|
|
2238
|
+
const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
|
|
2239
|
+
replaceKernelEntry(
|
|
2240
|
+
prefillProjectionKey,
|
|
2241
|
+
deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2242
|
+
?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
|
|
2243
|
+
?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
|
|
2244
|
+
);
|
|
2245
|
+
|
|
2246
|
+
const replacePrefillAttentionEntries = (entries) => {
|
|
2247
|
+
for (const entry of entries || []) {
|
|
2248
|
+
if (Array.isArray(entry)) {
|
|
2249
|
+
if (entry[0] !== 'attention') {
|
|
2250
|
+
continue;
|
|
2251
|
+
}
|
|
2252
|
+
replaceKernelEntry(
|
|
2253
|
+
entry[1],
|
|
2254
|
+
deriveF16AttentionKernelEntry(result.kernels[entry[1]])
|
|
2255
|
+
);
|
|
2256
|
+
continue;
|
|
2257
|
+
}
|
|
2258
|
+
if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
|
|
2259
|
+
replacePrefillAttentionEntries(entry.steps);
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2262
|
+
};
|
|
2263
|
+
replacePrefillAttentionEntries(result.prefill);
|
|
2264
|
+
|
|
2265
|
+
const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
|
|
2266
|
+
const finalNormKey = finalNormStep?.[1] ?? null;
|
|
2267
|
+
const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
|
|
2268
|
+
if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
|
|
2269
|
+
replaceKernelEntry(
|
|
2270
|
+
finalNormKey,
|
|
2271
|
+
deriveKernelEntryWithPrecision(
|
|
2272
|
+
deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
|
|
2273
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2274
|
+
)
|
|
2275
|
+
);
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
|
|
2279
|
+
const lmHeadKey = lmHeadStep?.[1] ?? null;
|
|
2280
|
+
replaceKernelEntry(
|
|
2281
|
+
lmHeadKey,
|
|
2282
|
+
deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
|
|
2283
|
+
);
|
|
2284
|
+
|
|
2285
|
+
const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
|
|
2286
|
+
const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
|
|
2287
|
+
const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
|
|
2288
|
+
if (
|
|
2289
|
+
lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
|
|
2290
|
+
|| lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
|
|
2291
|
+
) {
|
|
2292
|
+
replaceKernelEntry(
|
|
2293
|
+
lmHeadPrefillKey,
|
|
2294
|
+
deriveKernelEntryWithPrecision(
|
|
2295
|
+
deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
|
|
2296
|
+
{ inputDtype: 'f16', outputDtype: 'f16' }
|
|
2297
|
+
)
|
|
2298
|
+
);
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
return changed ? result : null;
|
|
2302
|
+
}
|
|
2303
|
+
|
|
1675
2304
|
// =============================================================================
|
|
1676
2305
|
// Composition
|
|
1677
2306
|
// =============================================================================
|
|
@@ -1698,6 +2327,34 @@ export function composeTransforms(...transforms) {
|
|
|
1698
2327
|
};
|
|
1699
2328
|
}
|
|
1700
2329
|
|
|
2330
|
+
// Session-only capability transform marker. The execution-v1 compiler applies
|
|
2331
|
+
// the matching runtime session patch; the graph itself is intentionally stable.
|
|
2332
|
+
export function disableRetainQ4KMaterialization() {
|
|
2333
|
+
return null;
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
/**
|
|
2337
|
+
* Fail-closed sentinel transform. A capability rule installs this when the
|
|
2338
|
+
* matched (modelId, runtime profile) combination is contradictory — for
|
|
2339
|
+
* example, an af32 manifest variant paired with a runtime profile that
|
|
2340
|
+
* demands f16 activations. The matcher reaches this transform only when the
|
|
2341
|
+
* earlier manifest-binding gate has been bypassed; throwing here keeps the
|
|
2342
|
+
* lane-confusion door shut at the capability layer too.
|
|
2343
|
+
*
|
|
2344
|
+
* @param {import('./execution-graph-transforms.js').ExecutionGraph} _graph
|
|
2345
|
+
* @param {import('./execution-graph-transforms.js').TransformContext} ctx
|
|
2346
|
+
* @returns {never}
|
|
2347
|
+
*/
|
|
2348
|
+
export function failClosedLaneMismatch(_graph, ctx) {
|
|
2349
|
+
const modelId = ctx?.modelId ?? 'unknown';
|
|
2350
|
+
const activationDtype = ctx?.activationDtype ?? 'unknown';
|
|
2351
|
+
throw new Error(
|
|
2352
|
+
`Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
|
|
2353
|
+
'The manifest variant tag is the lane identity — load the manifest variant ' +
|
|
2354
|
+
'whose compute lane matches the runtime profile.'
|
|
2355
|
+
);
|
|
2356
|
+
}
|
|
2357
|
+
|
|
1701
2358
|
// =============================================================================
|
|
1702
2359
|
// Registry
|
|
1703
2360
|
// =============================================================================
|
|
@@ -1709,6 +2366,7 @@ export const TRANSFORMS = Object.freeze({
|
|
|
1709
2366
|
widenToF32Activations,
|
|
1710
2367
|
widenToF32CorrectnessFallback,
|
|
1711
2368
|
swapPrefillAttention,
|
|
2369
|
+
useHead256SmallPrefillAttention,
|
|
1712
2370
|
useHead256PrefillAttention,
|
|
1713
2371
|
widenProjectionWeightsToF32,
|
|
1714
2372
|
remapDenseQ4KPrefillToQ4Native,
|
|
@@ -1718,7 +2376,13 @@ export const TRANSFORMS = Object.freeze({
|
|
|
1718
2376
|
remapQ4KDecodeAttentionToGemv,
|
|
1719
2377
|
remapQ4KDecodeAttentionToFusedQ4KGemv,
|
|
1720
2378
|
remapQ4KDecodeFFNToGemv,
|
|
2379
|
+
disableRetainQ4KMaterialization,
|
|
1721
2380
|
useQwenF16PrimaryMatmuls,
|
|
2381
|
+
useQwen36F16Activations,
|
|
1722
2382
|
useQwenDecodeF16Matmuls,
|
|
2383
|
+
useGemma4Int4PleSelectiveF16Decode,
|
|
2384
|
+
useGemma431BTextF16Activations,
|
|
2385
|
+
useGemma4Int4PleAf16Activations,
|
|
2386
|
+
failClosedLaneMismatch,
|
|
1723
2387
|
composeTransforms,
|
|
1724
2388
|
});
|