doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Fused Q4_K Matmul Kernel - WideTile register pattern (F16 A, Q4_K B, F16 C)
|
|
2
|
+
//
|
|
3
|
+
// W4A16 twin of fused_matmul_q4_widetile_f16.wgsl. The tiling and Q4_K
|
|
4
|
+
// dequantization path are identical, but the activation buffer is f16 so an
|
|
5
|
+
// execution-v1 f16 activation lane can keep prefill projections in f16 without
|
|
6
|
+
// an implicit widening cast.
|
|
7
|
+
|
|
8
|
+
enable f16;
|
|
9
|
+
|
|
10
|
+
const QK_K: u32 = 256u;
|
|
11
|
+
const SUBBLOCK_SIZE: u32 = 32u;
|
|
12
|
+
const NUM_SUBBLOCK_PAIRS: u32 = 4u;
|
|
13
|
+
const U32S_PER_PAIR: u32 = 8u;
|
|
14
|
+
|
|
15
|
+
override TILE_M: u32 = 4u;
|
|
16
|
+
override TILE_N: u32 = 256u;
|
|
17
|
+
|
|
18
|
+
const MAX_TILE_M: u32 = 8u;
|
|
19
|
+
|
|
20
|
+
struct Uniforms {
|
|
21
|
+
M: u32,
|
|
22
|
+
N: u32,
|
|
23
|
+
K: u32,
|
|
24
|
+
alpha: f32,
|
|
25
|
+
num_blocks_per_row: u32,
|
|
26
|
+
_pad0: u32,
|
|
27
|
+
_pad1: u32,
|
|
28
|
+
_pad2: u32,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
struct Q4KBlock {
|
|
32
|
+
d_dmin: u32,
|
|
33
|
+
scales: array<u32, 3>,
|
|
34
|
+
qs: array<u32, 32>,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
38
|
+
@group(0) @binding(1) var<storage, read> A: array<f16>;
|
|
39
|
+
@group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
|
|
40
|
+
@group(0) @binding(4) var<storage, read_write> C: array<f16>;
|
|
41
|
+
|
|
42
|
+
var<workgroup> tileA: array<f16, MAX_TILE_M * QK_K>;
|
|
43
|
+
|
|
44
|
+
fn unpack_f16_lo(packed: u32) -> f16 { return f16(unpack2x16float(packed).x); }
|
|
45
|
+
fn unpack_f16_hi(packed: u32) -> f16 { return f16(unpack2x16float(packed).y); }
|
|
46
|
+
|
|
47
|
+
fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
|
|
48
|
+
let word_idx = byte_idx / 4u;
|
|
49
|
+
let byte_in_word = byte_idx % 4u;
|
|
50
|
+
return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
|
|
54
|
+
var sc: u32;
|
|
55
|
+
var mn: u32;
|
|
56
|
+
if (j < 4u) {
|
|
57
|
+
sc = get_scale_byte(scales, j) & 63u;
|
|
58
|
+
mn = get_scale_byte(scales, j + 4u) & 63u;
|
|
59
|
+
} else {
|
|
60
|
+
let q_j = get_scale_byte(scales, j + 4u);
|
|
61
|
+
let q_lo = get_scale_byte(scales, j - 4u);
|
|
62
|
+
let q_hi = get_scale_byte(scales, j);
|
|
63
|
+
sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
|
|
64
|
+
mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
|
|
65
|
+
}
|
|
66
|
+
return vec2<u32>(sc, mn);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f16> {
|
|
70
|
+
let a_row_base = row_local * QK_K;
|
|
71
|
+
return vec4<f16>(
|
|
72
|
+
tileA[a_row_base + base_elem + 0u],
|
|
73
|
+
tileA[a_row_base + base_elem + 1u],
|
|
74
|
+
tileA[a_row_base + base_elem + 2u],
|
|
75
|
+
tileA[a_row_base + base_elem + 3u]
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
@compute @workgroup_size(TILE_N, 1, 1)
|
|
80
|
+
fn main(
|
|
81
|
+
@builtin(local_invocation_id) lid: vec3<u32>,
|
|
82
|
+
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
83
|
+
) {
|
|
84
|
+
if (TILE_M > MAX_TILE_M) { return; }
|
|
85
|
+
|
|
86
|
+
let col_local = lid.x;
|
|
87
|
+
let row_base = wg_id.y * TILE_M;
|
|
88
|
+
let col_base = wg_id.x * TILE_N;
|
|
89
|
+
let col = col_base + col_local;
|
|
90
|
+
|
|
91
|
+
var results: array<f16, MAX_TILE_M>;
|
|
92
|
+
for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
|
|
93
|
+
results[i] = f16(0.0);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
let num_blocks = u.num_blocks_per_row;
|
|
97
|
+
|
|
98
|
+
for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
|
|
99
|
+
let total_a_elems = TILE_M * QK_K;
|
|
100
|
+
for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
|
|
101
|
+
let load_row_local = idx / QK_K;
|
|
102
|
+
let load_k = idx % QK_K;
|
|
103
|
+
let global_row = row_base + load_row_local;
|
|
104
|
+
let global_k = b * QK_K + load_k;
|
|
105
|
+
if (global_row < u.M && global_k < u.K) {
|
|
106
|
+
tileA[idx] = A[global_row * u.K + global_k];
|
|
107
|
+
} else {
|
|
108
|
+
tileA[idx] = f16(0.0);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
workgroupBarrier();
|
|
112
|
+
|
|
113
|
+
if (col < u.N) {
|
|
114
|
+
let block = B_q4k[col * num_blocks + b];
|
|
115
|
+
let d = unpack_f16_lo(block.d_dmin);
|
|
116
|
+
let dmin = unpack_f16_hi(block.d_dmin);
|
|
117
|
+
|
|
118
|
+
for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
|
|
119
|
+
let sm_even = get_scale_min_k4(block.scales, 2u * p);
|
|
120
|
+
let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
|
|
121
|
+
let scale_even = d * f16(sm_even.x);
|
|
122
|
+
let min_even_scalar = dmin * f16(sm_even.y);
|
|
123
|
+
let scale_odd = d * f16(sm_odd.x);
|
|
124
|
+
let min_odd_scalar = dmin * f16(sm_odd.y);
|
|
125
|
+
let min_even = vec4<f16>(min_even_scalar);
|
|
126
|
+
let min_odd = vec4<f16>(min_odd_scalar);
|
|
127
|
+
|
|
128
|
+
let pair_base_even = p * 64u;
|
|
129
|
+
let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
|
|
130
|
+
let qs_base = p * U32S_PER_PAIR;
|
|
131
|
+
|
|
132
|
+
for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
|
|
133
|
+
let packed = block.qs[qs_base + uidx];
|
|
134
|
+
let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
|
|
135
|
+
let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
|
|
136
|
+
let w_even = scale_even * vec4<f16>(lower_u32) - min_even;
|
|
137
|
+
let w_odd = scale_odd * vec4<f16>(upper_u32) - min_odd;
|
|
138
|
+
|
|
139
|
+
let elem_even = pair_base_even + uidx * 4u;
|
|
140
|
+
let elem_odd = pair_base_odd + uidx * 4u;
|
|
141
|
+
|
|
142
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
143
|
+
let a_even = load_a_vec4(m, elem_even);
|
|
144
|
+
let a_odd = load_a_vec4(m, elem_odd);
|
|
145
|
+
results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
workgroupBarrier();
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (col < u.N) {
|
|
154
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
155
|
+
let row = row_base + m;
|
|
156
|
+
if (row < u.M) {
|
|
157
|
+
C[row * u.N + col] = results[m] * f16(u.alpha);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
// Fused Q4_K WideTile Matmul + Residual — F32 A, Q4_K B, F32 C
|
|
2
|
+
//
|
|
3
|
+
// Variant of fused_matmul_q4_widetile.wgsl that fuses the downstream
|
|
4
|
+
// residual add into the epilogue:
|
|
5
|
+
//
|
|
6
|
+
// output[m, n] = sum_k(input[m, k] * dequant(B_q4k)[k, n]) + residual[m, n]
|
|
7
|
+
//
|
|
8
|
+
// Targets the o_proj + attn_residual and ffn_down + ffn_residual sites
|
|
9
|
+
// where the subsequent residual add is otherwise a separate dispatch that
|
|
10
|
+
// pays the full ~0.85 ms/dispatch bubble cost on Dawn/Vulkan for ~200 µs
|
|
11
|
+
// of actual elementwise compute.
|
|
12
|
+
//
|
|
13
|
+
// All register-tiling, cooperative tileA load, unpack4xU8 dequant, and
|
|
14
|
+
// online-tile geometry match the base WideTile kernel. Only the final
|
|
15
|
+
// write differs. See fused_matmul_q4_widetile.wgsl for full design notes.
|
|
16
|
+
|
|
17
|
+
enable f16;
|
|
18
|
+
|
|
19
|
+
const QK_K: u32 = 256u;
|
|
20
|
+
const SUBBLOCK_SIZE: u32 = 32u;
|
|
21
|
+
const NUM_SUBBLOCK_PAIRS: u32 = 4u;
|
|
22
|
+
const U32S_PER_PAIR: u32 = 8u;
|
|
23
|
+
|
|
24
|
+
override TILE_M: u32 = 4u;
|
|
25
|
+
override TILE_N: u32 = 256u;
|
|
26
|
+
|
|
27
|
+
const MAX_TILE_M: u32 = 8u;
|
|
28
|
+
|
|
29
|
+
struct Uniforms {
|
|
30
|
+
M: u32,
|
|
31
|
+
N: u32,
|
|
32
|
+
K: u32,
|
|
33
|
+
alpha: f32,
|
|
34
|
+
num_blocks_per_row: u32,
|
|
35
|
+
_pad0: u32,
|
|
36
|
+
_pad1: u32,
|
|
37
|
+
_pad2: u32,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
struct Q4KBlock {
|
|
41
|
+
d_dmin: u32,
|
|
42
|
+
scales: array<u32, 3>,
|
|
43
|
+
qs: array<u32, 32>,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
47
|
+
@group(0) @binding(1) var<storage, read> A: array<f32>;
|
|
48
|
+
@group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
|
|
49
|
+
@group(0) @binding(3) var<storage, read_write> C: array<f32>;
|
|
50
|
+
@group(0) @binding(4) var<storage, read> residual: array<f32>;
|
|
51
|
+
|
|
52
|
+
var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
|
|
53
|
+
|
|
54
|
+
fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
|
|
55
|
+
fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
|
|
56
|
+
|
|
57
|
+
fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
|
|
58
|
+
let word_idx = byte_idx / 4u;
|
|
59
|
+
let byte_in_word = byte_idx % 4u;
|
|
60
|
+
return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
|
|
64
|
+
var sc: u32;
|
|
65
|
+
var mn: u32;
|
|
66
|
+
if (j < 4u) {
|
|
67
|
+
sc = get_scale_byte(scales, j) & 63u;
|
|
68
|
+
mn = get_scale_byte(scales, j + 4u) & 63u;
|
|
69
|
+
} else {
|
|
70
|
+
let q_j = get_scale_byte(scales, j + 4u);
|
|
71
|
+
let q_lo = get_scale_byte(scales, j - 4u);
|
|
72
|
+
let q_hi = get_scale_byte(scales, j);
|
|
73
|
+
sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
|
|
74
|
+
mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
|
|
75
|
+
}
|
|
76
|
+
return vec2<u32>(sc, mn);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
|
|
80
|
+
let a_row_base = row_local * QK_K;
|
|
81
|
+
return vec4<f32>(
|
|
82
|
+
tileA[a_row_base + base_elem + 0u],
|
|
83
|
+
tileA[a_row_base + base_elem + 1u],
|
|
84
|
+
tileA[a_row_base + base_elem + 2u],
|
|
85
|
+
tileA[a_row_base + base_elem + 3u]
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
@compute @workgroup_size(TILE_N, 1, 1)
|
|
90
|
+
fn main(
|
|
91
|
+
@builtin(local_invocation_id) lid: vec3<u32>,
|
|
92
|
+
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
93
|
+
) {
|
|
94
|
+
if (TILE_M > MAX_TILE_M) { return; }
|
|
95
|
+
|
|
96
|
+
let col_local = lid.x;
|
|
97
|
+
let row_base = wg_id.y * TILE_M;
|
|
98
|
+
let col_base = wg_id.x * TILE_N;
|
|
99
|
+
let col = col_base + col_local;
|
|
100
|
+
|
|
101
|
+
var results: array<f32, MAX_TILE_M>;
|
|
102
|
+
for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
|
|
103
|
+
results[i] = 0.0;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
let num_blocks = u.num_blocks_per_row;
|
|
107
|
+
|
|
108
|
+
for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
|
|
109
|
+
let total_a_elems = TILE_M * QK_K;
|
|
110
|
+
for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
|
|
111
|
+
let load_row_local = idx / QK_K;
|
|
112
|
+
let load_k = idx % QK_K;
|
|
113
|
+
let global_row = row_base + load_row_local;
|
|
114
|
+
let global_k = b * QK_K + load_k;
|
|
115
|
+
if (global_row < u.M && global_k < u.K) {
|
|
116
|
+
tileA[idx] = A[global_row * u.K + global_k];
|
|
117
|
+
} else {
|
|
118
|
+
tileA[idx] = 0.0;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
workgroupBarrier();
|
|
122
|
+
|
|
123
|
+
if (col < u.N) {
|
|
124
|
+
let block = B_q4k[col * num_blocks + b];
|
|
125
|
+
let d = unpack_f16_lo(block.d_dmin);
|
|
126
|
+
let dmin = unpack_f16_hi(block.d_dmin);
|
|
127
|
+
|
|
128
|
+
for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
|
|
129
|
+
let sm_even = get_scale_min_k4(block.scales, 2u * p);
|
|
130
|
+
let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
|
|
131
|
+
let scale_even = d * f32(sm_even.x);
|
|
132
|
+
let min_even_scalar = dmin * f32(sm_even.y);
|
|
133
|
+
let scale_odd = d * f32(sm_odd.x);
|
|
134
|
+
let min_odd_scalar = dmin * f32(sm_odd.y);
|
|
135
|
+
let min_even = vec4<f32>(min_even_scalar);
|
|
136
|
+
let min_odd = vec4<f32>(min_odd_scalar);
|
|
137
|
+
|
|
138
|
+
let pair_base_even = p * 64u;
|
|
139
|
+
let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
|
|
140
|
+
let qs_base = p * U32S_PER_PAIR;
|
|
141
|
+
|
|
142
|
+
for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
|
|
143
|
+
let packed = block.qs[qs_base + uidx];
|
|
144
|
+
let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
|
|
145
|
+
let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
|
|
146
|
+
let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
|
|
147
|
+
let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
|
|
148
|
+
|
|
149
|
+
let elem_even = pair_base_even + uidx * 4u;
|
|
150
|
+
let elem_odd = pair_base_odd + uidx * 4u;
|
|
151
|
+
|
|
152
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
153
|
+
let a_even = load_a_vec4(m, elem_even);
|
|
154
|
+
let a_odd = load_a_vec4(m, elem_odd);
|
|
155
|
+
results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
workgroupBarrier();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ===== Epilogue: matmul_result * alpha + residual =====
|
|
164
|
+
if (col < u.N) {
|
|
165
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
166
|
+
let row = row_base + m;
|
|
167
|
+
if (row < u.M) {
|
|
168
|
+
let offset = row * u.N + col;
|
|
169
|
+
C[offset] = results[m] * u.alpha + residual[offset];
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/fused_matmul_rmsnorm.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Fused GEMV + RMSNorm Kernel (F16 variant)
|
|
4
4
|
//
|
|
5
5
|
// For decode (M=1), combines the down projection matmul with RMSNorm in a single kernel:
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// Fused RMSNorm + Q4_K WideTile Matmul — F32 A, Q4_K B, F32 C
|
|
2
|
+
//
|
|
3
|
+
// Prologue fuses RMSNorm into the WideTile matmul:
|
|
4
|
+
// normed[m, k] = (input[m, k] / rms(input[m, :])) * norm_weight[k]
|
|
5
|
+
// output[m, n] = sum_k(normed[m, k] * dequant(B_q4k)[k, n])
|
|
6
|
+
//
|
|
7
|
+
// Targets the two Gemma-style pre-matmul norms:
|
|
8
|
+
// input_norm → {q,k,v}_proj (each q/k/v call runs norm internally)
|
|
9
|
+
// pre_feedforward_norm → {gate,up}_proj
|
|
10
|
+
//
|
|
11
|
+
// Redundant norm work across q/k/v is negligible (norm: O(K) per row;
|
|
12
|
+
// matmul: O(K*N_per_WG) per row). Each saved standalone rmsnorm dispatch
|
|
13
|
+
// cuts ~0.85 ms of Dawn/Vulkan bubble — the target.
|
|
14
|
+
//
|
|
15
|
+
// Weight offset: when `RMS_NORM_OFFSET = true`, the stored norm weight
|
|
16
|
+
// encodes `(weight - 1.0)` (Gemma family convention). This kernel reads
|
|
17
|
+
// stored_w and applies `1.0 + stored_w` at use-site.
|
|
18
|
+
//
|
|
19
|
+
// Shared memory: tileA[TILE_M][QK_K] f32 = 4 KB per WG block (sliced
|
|
20
|
+
// across K in the existing WideTile loop). The full-row normed tensor
|
|
21
|
+
// is also cached in shared at `row_sq_sum` + `rms` derived per row;
|
|
22
|
+
// we compute sum-of-squares across K chunks in the same outer loop.
|
|
23
|
+
//
|
|
24
|
+
// Two-pass structure: pass 1 accumulates sum_sq across K, computes rms;
|
|
25
|
+
// pass 2 reuses the same tileA buffer to multiply normed*weight and
|
|
26
|
+
// dequant+matmul. Pass 2 reads K in the same block iteration order.
|
|
27
|
+
//
|
|
28
|
+
// Adapted from the iter-8 WideTile design.
|
|
29
|
+
|
|
30
|
+
enable f16;
|
|
31
|
+
|
|
32
|
+
const QK_K: u32 = 256u;
|
|
33
|
+
const SUBBLOCK_SIZE: u32 = 32u;
|
|
34
|
+
const NUM_SUBBLOCK_PAIRS: u32 = 4u;
|
|
35
|
+
const U32S_PER_PAIR: u32 = 8u;
|
|
36
|
+
|
|
37
|
+
override TILE_M: u32 = 4u;
|
|
38
|
+
override TILE_N: u32 = 256u;
|
|
39
|
+
override RMS_NORM_OFFSET: bool = false;
|
|
40
|
+
override WEIGHT_IS_F16: bool = false;
|
|
41
|
+
|
|
42
|
+
const MAX_TILE_M: u32 = 8u;
|
|
43
|
+
const MAX_TILE_N: u32 = 256u;
|
|
44
|
+
|
|
45
|
+
struct Uniforms {
|
|
46
|
+
M: u32,
|
|
47
|
+
N: u32,
|
|
48
|
+
K: u32,
|
|
49
|
+
alpha: f32,
|
|
50
|
+
num_blocks_per_row: u32,
|
|
51
|
+
eps: f32,
|
|
52
|
+
_pad0: u32,
|
|
53
|
+
_pad1: u32,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
struct Q4KBlock {
|
|
57
|
+
d_dmin: u32,
|
|
58
|
+
scales: array<u32, 3>,
|
|
59
|
+
qs: array<u32, 32>,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
63
|
+
@group(0) @binding(1) var<storage, read> A: array<f32>;
|
|
64
|
+
@group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
|
|
65
|
+
@group(0) @binding(3) var<storage, read_write> C: array<f32>;
|
|
66
|
+
@group(0) @binding(4) var<storage, read> norm_weight: array<u32>;
|
|
67
|
+
|
|
68
|
+
var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
|
|
69
|
+
// Per-row reduction buffer for pass 1: [TILE_M][TILE_N] partial sum-of-squares.
|
|
70
|
+
// Each thread writes its per-row partial, then workgroup-parallel tree reduction
|
|
71
|
+
// collapses to shared_sum_sq[m][0]. Derived row_rms_recip is cached for pass 2.
|
|
72
|
+
var<workgroup> shared_sum_sq: array<array<f32, MAX_TILE_N>, MAX_TILE_M>;
|
|
73
|
+
var<workgroup> row_rms_recip: array<f32, MAX_TILE_M>;
|
|
74
|
+
|
|
75
|
+
fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
|
|
76
|
+
fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
|
|
77
|
+
|
|
78
|
+
// norm_weight binding is declared as array<u32> so it can carry either packed
|
|
79
|
+
// f32 values (bitcast) or packed f16 pairs (Gemma-family hidden weights).
|
|
80
|
+
// Mirrors the load_weight pattern in rmsnorm.wgsl.
|
|
81
|
+
fn load_norm_weight(idx: u32) -> f32 {
|
|
82
|
+
if (WEIGHT_IS_F16) {
|
|
83
|
+
let packed = norm_weight[idx >> 1u];
|
|
84
|
+
let pair = unpack2x16float(packed);
|
|
85
|
+
return select(pair.x, pair.y, (idx & 1u) == 1u);
|
|
86
|
+
}
|
|
87
|
+
return bitcast<f32>(norm_weight[idx]);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
|
|
91
|
+
let word_idx = byte_idx / 4u;
|
|
92
|
+
let byte_in_word = byte_idx % 4u;
|
|
93
|
+
return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
|
|
97
|
+
var sc: u32;
|
|
98
|
+
var mn: u32;
|
|
99
|
+
if (j < 4u) {
|
|
100
|
+
sc = get_scale_byte(scales, j) & 63u;
|
|
101
|
+
mn = get_scale_byte(scales, j + 4u) & 63u;
|
|
102
|
+
} else {
|
|
103
|
+
let q_j = get_scale_byte(scales, j + 4u);
|
|
104
|
+
let q_lo = get_scale_byte(scales, j - 4u);
|
|
105
|
+
let q_hi = get_scale_byte(scales, j);
|
|
106
|
+
sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
|
|
107
|
+
mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
|
|
108
|
+
}
|
|
109
|
+
return vec2<u32>(sc, mn);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
fn load_normed_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
|
|
113
|
+
// tileA at this point holds normed values (pass 2). The rms scaling
|
|
114
|
+
// and norm_weight multiplication have already been applied when tileA
|
|
115
|
+
// was populated for this block. See $main pass-2 section.
|
|
116
|
+
let a_row_base = row_local * QK_K;
|
|
117
|
+
return vec4<f32>(
|
|
118
|
+
tileA[a_row_base + base_elem + 0u],
|
|
119
|
+
tileA[a_row_base + base_elem + 1u],
|
|
120
|
+
tileA[a_row_base + base_elem + 2u],
|
|
121
|
+
tileA[a_row_base + base_elem + 3u]
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
@compute @workgroup_size(TILE_N, 1, 1)
|
|
126
|
+
fn main(
|
|
127
|
+
@builtin(local_invocation_id) lid: vec3<u32>,
|
|
128
|
+
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
129
|
+
) {
|
|
130
|
+
if (TILE_M > MAX_TILE_M) { return; }
|
|
131
|
+
|
|
132
|
+
let col_local = lid.x;
|
|
133
|
+
let row_base = wg_id.y * TILE_M;
|
|
134
|
+
let col_base = wg_id.x * TILE_N;
|
|
135
|
+
let col = col_base + col_local;
|
|
136
|
+
|
|
137
|
+
var results: array<f32, MAX_TILE_M>;
|
|
138
|
+
for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
|
|
139
|
+
results[i] = 0.0;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
let num_blocks = u.num_blocks_per_row;
|
|
143
|
+
|
|
144
|
+
// ===== PASS 1: workgroup-parallel sum-of-squares per row across all K =====
|
|
145
|
+
// Each of TILE_N threads accumulates a per-thread partial across blocks,
|
|
146
|
+
// then a tree reduction over TILE_N lanes finalises the sum per row. The
|
|
147
|
+
// prior serialised 4-thread reduction was the iter-21 perf wall; this
|
|
148
|
+
// restores full workgroup parallelism (log2(TILE_N)=8 reduction steps).
|
|
149
|
+
var local_sum_sq: array<f32, MAX_TILE_M>;
|
|
150
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
151
|
+
local_sum_sq[m] = 0.0;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Streaming accumulation: thread col_local handles k_in_block == col_local
|
|
155
|
+
// across every block. QK_K (256) == TILE_N (256), so the mapping is 1:1
|
|
156
|
+
// and every lane picks up a K-value per block without serialisation.
|
|
157
|
+
for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
|
|
158
|
+
let global_k = b * QK_K + col_local;
|
|
159
|
+
if (global_k < u.K) {
|
|
160
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
161
|
+
let global_row = row_base + m;
|
|
162
|
+
if (global_row < u.M) {
|
|
163
|
+
let v = A[global_row * u.K + global_k];
|
|
164
|
+
local_sum_sq[m] = local_sum_sq[m] + v * v;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Publish partials into shared memory for tree reduction.
|
|
171
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
172
|
+
shared_sum_sq[m][col_local] = local_sum_sq[m];
|
|
173
|
+
}
|
|
174
|
+
workgroupBarrier();
|
|
175
|
+
|
|
176
|
+
// Tree reduction over TILE_N lanes. All TILE_M rows are reduced in
|
|
177
|
+
// parallel at each stride — 8 barriers total (log2(256)).
|
|
178
|
+
for (var stride: u32 = TILE_N / 2u; stride > 0u; stride = stride >> 1u) {
|
|
179
|
+
if (col_local < stride) {
|
|
180
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
181
|
+
shared_sum_sq[m][col_local] =
|
|
182
|
+
shared_sum_sq[m][col_local]
|
|
183
|
+
+ shared_sum_sq[m][col_local + stride];
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
workgroupBarrier();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Derive 1/rms per row. One thread per row reads the reduced total.
|
|
190
|
+
if (col_local < TILE_M) {
|
|
191
|
+
let mean_sq = shared_sum_sq[col_local][0] / f32(u.K);
|
|
192
|
+
row_rms_recip[col_local] = 1.0 / sqrt(mean_sq + u.eps);
|
|
193
|
+
}
|
|
194
|
+
workgroupBarrier();
|
|
195
|
+
|
|
196
|
+
// ===== PASS 2: reload A, apply rmsnorm + weight, matmul =====
|
|
197
|
+
for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
|
|
198
|
+
// Cooperative load of tileA raw (re-read from global — cheap vs
|
|
199
|
+
// matmul work), apply norm in place, then matmul. Applying norm
|
|
200
|
+
// in the same cooperative load avoids an extra barrier.
|
|
201
|
+
let total_a_elems = TILE_M * QK_K;
|
|
202
|
+
for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
|
|
203
|
+
let load_row_local = idx / QK_K;
|
|
204
|
+
let load_k = idx % QK_K;
|
|
205
|
+
let global_row = row_base + load_row_local;
|
|
206
|
+
let global_k = b * QK_K + load_k;
|
|
207
|
+
var raw: f32 = 0.0;
|
|
208
|
+
if (global_row < u.M && global_k < u.K) {
|
|
209
|
+
raw = A[global_row * u.K + global_k];
|
|
210
|
+
}
|
|
211
|
+
// rmsnorm: x * (1/rms)
|
|
212
|
+
var normed = raw * row_rms_recip[load_row_local];
|
|
213
|
+
// weight: multiply by norm_weight (with Gemma (1+w) offset).
|
|
214
|
+
if (global_k < u.K) {
|
|
215
|
+
let stored_w = load_norm_weight(global_k);
|
|
216
|
+
let effective_w = select(stored_w, 1.0 + stored_w, RMS_NORM_OFFSET);
|
|
217
|
+
normed = normed * effective_w;
|
|
218
|
+
}
|
|
219
|
+
tileA[idx] = normed;
|
|
220
|
+
}
|
|
221
|
+
workgroupBarrier();
|
|
222
|
+
|
|
223
|
+
if (col < u.N) {
|
|
224
|
+
let block = B_q4k[col * num_blocks + b];
|
|
225
|
+
let d = unpack_f16_lo(block.d_dmin);
|
|
226
|
+
let dmin = unpack_f16_hi(block.d_dmin);
|
|
227
|
+
|
|
228
|
+
for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
|
|
229
|
+
let sm_even = get_scale_min_k4(block.scales, 2u * p);
|
|
230
|
+
let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
|
|
231
|
+
let scale_even = d * f32(sm_even.x);
|
|
232
|
+
let min_even_scalar = dmin * f32(sm_even.y);
|
|
233
|
+
let scale_odd = d * f32(sm_odd.x);
|
|
234
|
+
let min_odd_scalar = dmin * f32(sm_odd.y);
|
|
235
|
+
let min_even = vec4<f32>(min_even_scalar);
|
|
236
|
+
let min_odd = vec4<f32>(min_odd_scalar);
|
|
237
|
+
|
|
238
|
+
let pair_base_even = p * 64u;
|
|
239
|
+
let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
|
|
240
|
+
let qs_base = p * U32S_PER_PAIR;
|
|
241
|
+
|
|
242
|
+
for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
|
|
243
|
+
let packed = block.qs[qs_base + uidx];
|
|
244
|
+
let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
|
|
245
|
+
let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
|
|
246
|
+
let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
|
|
247
|
+
let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
|
|
248
|
+
|
|
249
|
+
let elem_even = pair_base_even + uidx * 4u;
|
|
250
|
+
let elem_odd = pair_base_odd + uidx * 4u;
|
|
251
|
+
|
|
252
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
253
|
+
let a_even = load_normed_a_vec4(m, elem_even);
|
|
254
|
+
let a_odd = load_normed_a_vec4(m, elem_odd);
|
|
255
|
+
results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
workgroupBarrier();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if (col < u.N) {
|
|
264
|
+
for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
|
|
265
|
+
let row = row_base + m;
|
|
266
|
+
if (row < u.M) {
|
|
267
|
+
C[row * u.N + col] = results[m] * u.alpha;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/fused_swiglu.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// fused_swiglu_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -35,13 +35,6 @@ export interface GatherOptions extends OutputBufferOptions {
|
|
|
35
35
|
indirectOffset?: number;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
/** Context for gather variant selection rules. */
|
|
39
|
-
export interface GatherSelectionContext {
|
|
40
|
-
useF16Input: boolean;
|
|
41
|
-
useF16Output: boolean;
|
|
42
|
-
useVec4: boolean;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
38
|
/**
|
|
46
39
|
* Run gather/embedding lookup
|
|
47
40
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/gather.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// gather_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|