doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -8,6 +8,7 @@ enable subgroups;
|
|
|
8
8
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
9
9
|
const MAX_SUBGROUPS: u32 = 256u;
|
|
10
10
|
const MAX_HEAD_DIM: u32 = 512u;
|
|
11
|
+
const NEG_INF = f16(-65504.0);
|
|
11
12
|
|
|
12
13
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
13
14
|
|
|
@@ -37,12 +38,12 @@ struct Uniforms {
|
|
|
37
38
|
@group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
|
|
38
39
|
@group(0) @binding(6) var<storage, read> page_table: array<u32>;
|
|
39
40
|
|
|
40
|
-
var<workgroup> shared_q: array<
|
|
41
|
-
var<workgroup> shared_scores: array<
|
|
42
|
-
var<workgroup> sg_max: array<
|
|
43
|
-
var<workgroup> sg_sum: array<
|
|
44
|
-
var<workgroup> global_max:
|
|
45
|
-
var<workgroup> global_sum:
|
|
41
|
+
var<workgroup> shared_q: array<f16, MAX_HEAD_DIM>;
|
|
42
|
+
var<workgroup> shared_scores: array<f16, MAX_WORKGROUP_SIZE>;
|
|
43
|
+
var<workgroup> sg_max: array<f16, MAX_SUBGROUPS>;
|
|
44
|
+
var<workgroup> sg_sum: array<f16, MAX_SUBGROUPS>;
|
|
45
|
+
var<workgroup> global_max: f16;
|
|
46
|
+
var<workgroup> global_sum: f16;
|
|
46
47
|
|
|
47
48
|
fn get_kv_head_idx(query_head_idx: u32) -> u32 {
|
|
48
49
|
let heads_per_kv = u.num_heads / u.num_kv_heads;
|
|
@@ -90,6 +91,8 @@ fn main(
|
|
|
90
91
|
let tid = local_id.x;
|
|
91
92
|
let head_dim = u.head_dim;
|
|
92
93
|
let kv_len = get_kv_len();
|
|
94
|
+
let scale = f16(u.scale);
|
|
95
|
+
let softcap = f16(u.attn_softcap);
|
|
93
96
|
let q_offset = head_idx * head_dim;
|
|
94
97
|
let out_dim0 = tid;
|
|
95
98
|
let out_dim1 = tid + WORKGROUP_SIZE;
|
|
@@ -119,17 +122,17 @@ fn main(
|
|
|
119
122
|
let kv_head_idx = get_kv_head_idx(head_idx);
|
|
120
123
|
|
|
121
124
|
if (has_out_dim0) {
|
|
122
|
-
shared_q[out_dim0] =
|
|
125
|
+
shared_q[out_dim0] = Q[q_offset + out_dim0];
|
|
123
126
|
}
|
|
124
127
|
if (has_out_dim1) {
|
|
125
|
-
shared_q[out_dim1] =
|
|
128
|
+
shared_q[out_dim1] = Q[q_offset + out_dim1];
|
|
126
129
|
}
|
|
127
130
|
workgroupBarrier();
|
|
128
131
|
|
|
129
|
-
var running_max:
|
|
130
|
-
var running_sum:
|
|
131
|
-
var out_accum0:
|
|
132
|
-
var out_accum1:
|
|
132
|
+
var running_max: f16 = NEG_INF;
|
|
133
|
+
var running_sum: f16 = f16(0.0);
|
|
134
|
+
var out_accum0: f16 = f16(0.0);
|
|
135
|
+
var out_accum1: f16 = f16(0.0);
|
|
133
136
|
|
|
134
137
|
var start_k: u32 = 0u;
|
|
135
138
|
if (u.sliding_window > 0u && kv_len > u.sliding_window) {
|
|
@@ -142,7 +145,7 @@ fn main(
|
|
|
142
145
|
let k_pos = k_start + tid;
|
|
143
146
|
let valid_k = k_pos < kv_len;
|
|
144
147
|
var masked = false;
|
|
145
|
-
var score:
|
|
148
|
+
var score: f16 = NEG_INF;
|
|
146
149
|
|
|
147
150
|
if (valid_k) {
|
|
148
151
|
let abs_key = u.kv_start + k_pos;
|
|
@@ -150,32 +153,32 @@ fn main(
|
|
|
150
153
|
if (!masked) {
|
|
151
154
|
let k_idx = get_kv_pos(k_pos);
|
|
152
155
|
let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
153
|
-
var dot:
|
|
156
|
+
var dot: f16 = f16(0.0);
|
|
154
157
|
for (var d: u32 = 0u; d < head_dim; d = d + 2u) {
|
|
155
158
|
let q0 = shared_q[d];
|
|
156
|
-
let k0 =
|
|
159
|
+
let k0 = K[k_offset + d];
|
|
157
160
|
dot = dot + q0 * k0;
|
|
158
161
|
if (d + 1u < head_dim) {
|
|
159
162
|
let q1 = shared_q[d + 1u];
|
|
160
|
-
let k1 =
|
|
163
|
+
let k1 = K[k_offset + d + 1u];
|
|
161
164
|
dot = dot + q1 * k1;
|
|
162
165
|
}
|
|
163
166
|
}
|
|
164
|
-
score = dot *
|
|
165
|
-
if (
|
|
166
|
-
score = tanh(score /
|
|
167
|
+
score = dot * scale;
|
|
168
|
+
if (softcap > f16(0.0)) {
|
|
169
|
+
score = tanh(score / softcap) * softcap;
|
|
167
170
|
}
|
|
168
171
|
}
|
|
169
172
|
}
|
|
170
173
|
|
|
171
|
-
|
|
174
|
+
let chunk_max = subgroupMax(score);
|
|
172
175
|
if (sg_tid == 0u && subgroup_id < num_subgroups) {
|
|
173
176
|
sg_max[subgroup_id] = chunk_max;
|
|
174
177
|
}
|
|
175
178
|
workgroupBarrier();
|
|
176
179
|
|
|
177
180
|
if (tid == 0u) {
|
|
178
|
-
var m:
|
|
181
|
+
var m: f16 = NEG_INF;
|
|
179
182
|
for (var s: u32 = 0u; s < num_subgroups; s++) {
|
|
180
183
|
m = max(m, sg_max[s]);
|
|
181
184
|
}
|
|
@@ -187,20 +190,20 @@ fn main(
|
|
|
187
190
|
let new_max = max(running_max, chunk_max_val);
|
|
188
191
|
let rescale = exp(running_max - new_max);
|
|
189
192
|
|
|
190
|
-
var exp_score:
|
|
193
|
+
var exp_score: f16 = f16(0.0);
|
|
191
194
|
if (valid_k && !masked) {
|
|
192
195
|
exp_score = exp(score - new_max);
|
|
193
196
|
}
|
|
194
197
|
shared_scores[tid] = exp_score;
|
|
195
198
|
|
|
196
|
-
|
|
199
|
+
let chunk_sum = subgroupAdd(exp_score);
|
|
197
200
|
if (sg_tid == 0u && subgroup_id < num_subgroups) {
|
|
198
201
|
sg_sum[subgroup_id] = chunk_sum;
|
|
199
202
|
}
|
|
200
203
|
workgroupBarrier();
|
|
201
204
|
|
|
202
205
|
if (tid == 0u) {
|
|
203
|
-
var s:
|
|
206
|
+
var s: f16 = f16(0.0);
|
|
204
207
|
for (var i: u32 = 0u; i < num_subgroups; i++) {
|
|
205
208
|
s = s + sg_sum[i];
|
|
206
209
|
}
|
|
@@ -221,21 +224,21 @@ fn main(
|
|
|
221
224
|
let v_idx = get_kv_pos(k_pos_inner);
|
|
222
225
|
let v_base = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
223
226
|
if (has_out_dim0) {
|
|
224
|
-
out_accum0 = out_accum0 + shared_scores[score_idx] *
|
|
227
|
+
out_accum0 = out_accum0 + shared_scores[score_idx] * V[v_base + out_dim0];
|
|
225
228
|
}
|
|
226
229
|
if (has_out_dim1) {
|
|
227
|
-
out_accum1 = out_accum1 + shared_scores[score_idx] *
|
|
230
|
+
out_accum1 = out_accum1 + shared_scores[score_idx] * V[v_base + out_dim1];
|
|
228
231
|
}
|
|
229
232
|
}
|
|
230
233
|
}
|
|
231
234
|
workgroupBarrier();
|
|
232
235
|
}
|
|
233
236
|
|
|
234
|
-
let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
|
|
237
|
+
let inv_sum = select(f16(0.0), f16(1.0) / running_sum, running_sum > f16(0.0));
|
|
235
238
|
if (has_out_dim0) {
|
|
236
|
-
output[q_offset + out_dim0] =
|
|
239
|
+
output[q_offset + out_dim0] = out_accum0 * inv_sum;
|
|
237
240
|
}
|
|
238
241
|
if (has_out_dim1) {
|
|
239
|
-
output[q_offset + out_dim1] =
|
|
242
|
+
output[q_offset + out_dim1] = out_accum1 * inv_sum;
|
|
240
243
|
}
|
|
241
244
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention_decode_online_f16.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Online Decode Attention Kernel (f32 Q + f16 KV + f32 output)
|
|
4
4
|
//
|
|
5
5
|
// Uses online softmax with subgroup reductions and chunked KV processing.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention_decode_paged_f16.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Paged Decode Attention Kernel (f16 KV)
|
|
4
4
|
//
|
|
5
5
|
// Q is f32, K/V are f16, output is f32. Uses page table indirection.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention_decode_tiered_f16.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Tiered Decode Attention Kernel (f32 Q + f16 KV)
|
|
4
4
|
//
|
|
5
5
|
// Hot ring buffer + cold paged cache with split-domain softmax.
|
|
@@ -1,63 +1,100 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
3
|
-
// Fused Multi-Head Attention Kernel
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
|
+
// Fused Multi-Head Attention Kernel
|
|
4
4
|
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
5
|
+
// Implements fused Q @ K^T → scale → mask → softmax → @ V
|
|
6
|
+
// Uses tiled/blocked approach to avoid materializing full attention matrix.
|
|
7
|
+
// Supports grouped query attention (GQA) where numKVHeads < numHeads.
|
|
8
|
+
//
|
|
9
|
+
// Based on Flash Attention principles adapted for WebGPU.
|
|
7
10
|
|
|
8
11
|
enable f16;
|
|
9
12
|
|
|
10
13
|
// Tile sizes for blocked attention
|
|
11
|
-
const MAX_BLOCK_SIZE: u32 =
|
|
14
|
+
const MAX_BLOCK_SIZE: u32 = 32u;
|
|
12
15
|
const MAX_HEAD_TILE: u32 = 64u;
|
|
13
16
|
const MAX_HEAD_DIM: u32 = 64u;
|
|
14
17
|
|
|
15
|
-
override BLOCK_SIZE: u32 =
|
|
16
|
-
override HEAD_TILE: u32 = 64u;
|
|
17
|
-
override WORKGROUP_SIZE: u32 =
|
|
18
|
+
override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
|
|
19
|
+
override HEAD_TILE: u32 = 64u; // Head dimension tile
|
|
20
|
+
override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
|
|
18
21
|
|
|
19
22
|
struct Uniforms {
|
|
20
|
-
num_heads: u32,
|
|
21
|
-
num_kv_heads: u32,
|
|
22
|
-
head_dim: u32,
|
|
23
|
-
seq_len: u32,
|
|
24
|
-
query_len: u32,
|
|
25
|
-
scale: f32,
|
|
26
|
-
is_causal: u32,
|
|
27
|
-
start_pos: u32,
|
|
28
|
-
attn_softcap: f32,
|
|
29
|
-
sliding_window: u32,
|
|
30
|
-
kv_len_source: u32,
|
|
23
|
+
num_heads: u32, // Number of query heads
|
|
24
|
+
num_kv_heads: u32, // Number of KV heads (for GQA)
|
|
25
|
+
head_dim: u32, // Dimension per head
|
|
26
|
+
seq_len: u32, // Current sequence length (for KV)
|
|
27
|
+
query_len: u32, // Query length (1 for decode, seq_len for prefill)
|
|
28
|
+
scale: f32, // 1/sqrt(head_dim)
|
|
29
|
+
is_causal: u32, // Apply causal mask (1 = yes)
|
|
30
|
+
start_pos: u32, // Absolute position offset for causal masking
|
|
31
|
+
attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
|
|
32
|
+
sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
|
|
33
|
+
kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
|
|
31
34
|
kv_start: u32,
|
|
32
35
|
page_size: u32,
|
|
33
36
|
kv_layout: u32,
|
|
34
|
-
|
|
37
|
+
bidirectional_span_start: u32,
|
|
38
|
+
bidirectional_span_length: u32,
|
|
39
|
+
_pad0: u32,
|
|
40
|
+
_pad1: u32,
|
|
41
|
+
_pad2: u32,
|
|
35
42
|
}
|
|
36
43
|
|
|
37
44
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
38
|
-
@group(0) @binding(1) var<storage, read> Q: array<f16>;
|
|
39
|
-
@group(0) @binding(2) var<storage, read> K: array<f16>;
|
|
40
|
-
@group(0) @binding(3) var<storage, read> V: array<f16>;
|
|
41
|
-
@group(0) @binding(4) var<storage, read_write> output: array<f16>;
|
|
45
|
+
@group(0) @binding(1) var<storage, read> Q: array<f16>; // [query_len, num_heads, head_dim]
|
|
46
|
+
@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
|
|
47
|
+
@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
|
|
48
|
+
@group(0) @binding(4) var<storage, read_write> output: array<f16>; // [query_len, num_heads, head_dim]
|
|
42
49
|
@group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
|
|
43
50
|
@group(0) @binding(6) var<storage, read> page_table: array<u32>;
|
|
44
51
|
|
|
52
|
+
// Shared memory for tiled computation
|
|
45
53
|
var<workgroup> shared_K: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
|
|
46
54
|
var<workgroup> shared_V: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
|
|
47
55
|
var<workgroup> shared_scores: array<f32, MAX_BLOCK_SIZE * MAX_BLOCK_SIZE>;
|
|
48
56
|
|
|
57
|
+
// Get KV head index for grouped query attention
|
|
49
58
|
fn get_kv_head_idx(query_head_idx: u32) -> u32 {
|
|
59
|
+
// GQA: multiple query heads share one KV head
|
|
50
60
|
let heads_per_kv = u.num_heads / u.num_kv_heads;
|
|
51
61
|
return query_head_idx / heads_per_kv;
|
|
52
62
|
}
|
|
53
63
|
|
|
64
|
+
fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
|
|
65
|
+
if (u.bidirectional_span_length == 0u) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
let span_start = u.bidirectional_span_start;
|
|
69
|
+
let span_end = span_start + u.bidirectional_span_length;
|
|
70
|
+
return abs_query >= span_start
|
|
71
|
+
&& abs_query < span_end
|
|
72
|
+
&& abs_key >= span_start
|
|
73
|
+
&& abs_key < span_end;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Check if position should be masked (causal + sliding window attention)
|
|
54
77
|
fn is_masked(query_pos: u32, key_pos: u32) -> bool {
|
|
78
|
+
// Compute absolute positions
|
|
55
79
|
let abs_query = query_pos + u.start_pos;
|
|
56
80
|
let abs_key = u.kv_start + key_pos;
|
|
57
|
-
|
|
81
|
+
|
|
82
|
+
// Causal mask: query can only attend to keys at same or earlier positions
|
|
83
|
+
if (u.is_causal != 0u && abs_key > abs_query) {
|
|
84
|
+
if (is_bidirectional_span_visible(abs_query, abs_key)) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Sliding window mask: query can only attend to keys within the window
|
|
91
|
+
// Key must be >= (query - window + 1), i.e., within the last `window` positions
|
|
58
92
|
if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
|
|
59
|
-
if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
93
|
+
if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
94
|
+
return true; // Key is too far in the past
|
|
95
|
+
}
|
|
60
96
|
}
|
|
97
|
+
|
|
61
98
|
return false;
|
|
62
99
|
}
|
|
63
100
|
|
|
@@ -82,9 +119,11 @@ fn get_kv_len() -> u32 {
|
|
|
82
119
|
return kv_len_buffer[0];
|
|
83
120
|
}
|
|
84
121
|
|
|
122
|
+
// Main attention kernel - one workgroup per (query_block, head)
|
|
123
|
+
// Workgroups are dispatched linearly as: numQueryBlocks * numHeads.
|
|
124
|
+
// head_idx and query_block_idx are derived from workgroup_id.x.
|
|
85
125
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
86
126
|
fn main(
|
|
87
|
-
@builtin(global_invocation_id) global_id: vec3<u32>,
|
|
88
127
|
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
89
128
|
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
90
129
|
) {
|
|
@@ -102,17 +141,21 @@ fn main(
|
|
|
102
141
|
let query_len = u.query_len;
|
|
103
142
|
let scale = u.scale;
|
|
104
143
|
|
|
144
|
+
// Query position this thread handles
|
|
105
145
|
let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
|
|
106
146
|
let valid_query = query_pos < query_len;
|
|
107
147
|
|
|
108
|
-
|
|
109
|
-
var
|
|
110
|
-
var
|
|
148
|
+
// Initialize online softmax accumulators
|
|
149
|
+
var m_i: f32 = -3.402823e+38; // -inf for max tracking
|
|
150
|
+
var l_i: f32 = 0.0; // Sum of exp(x - max)
|
|
151
|
+
var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
|
|
111
152
|
|
|
153
|
+
// Initialize accumulator
|
|
112
154
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
113
155
|
acc[d] = 0.0;
|
|
114
156
|
}
|
|
115
157
|
|
|
158
|
+
// Load query for this thread into registers
|
|
116
159
|
var q_local: array<f32, MAX_HEAD_DIM>;
|
|
117
160
|
if (valid_query) {
|
|
118
161
|
let q_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
|
|
@@ -121,11 +164,13 @@ fn main(
|
|
|
121
164
|
}
|
|
122
165
|
}
|
|
123
166
|
|
|
167
|
+
// Process key-value blocks
|
|
124
168
|
let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
|
|
125
169
|
|
|
126
170
|
for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
|
|
127
171
|
let kv_block_start = kv_block * BLOCK_SIZE;
|
|
128
172
|
|
|
173
|
+
// Collaborative load of K block into shared memory
|
|
129
174
|
let k_load_idx = kv_block_start + thread_idx;
|
|
130
175
|
if (k_load_idx < seq_len) {
|
|
131
176
|
let k_idx = get_kv_pos(k_load_idx);
|
|
@@ -139,6 +184,7 @@ fn main(
|
|
|
139
184
|
}
|
|
140
185
|
}
|
|
141
186
|
|
|
187
|
+
// Load V block
|
|
142
188
|
let v_load_idx = kv_block_start + thread_idx;
|
|
143
189
|
if (v_load_idx < seq_len) {
|
|
144
190
|
let v_idx = get_kv_pos(v_load_idx);
|
|
@@ -154,20 +200,26 @@ fn main(
|
|
|
154
200
|
|
|
155
201
|
workgroupBarrier();
|
|
156
202
|
|
|
203
|
+
// Compute attention scores for this block
|
|
157
204
|
if (valid_query) {
|
|
205
|
+
// Find max in this block (for numerical stability)
|
|
158
206
|
var block_max: f32 = -3.402823e+38;
|
|
159
207
|
|
|
160
208
|
for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
|
|
161
209
|
let key_pos = kv_block_start + k;
|
|
162
210
|
if (key_pos >= seq_len) { continue; }
|
|
211
|
+
|
|
212
|
+
// Check causal mask
|
|
163
213
|
if (is_masked(query_pos, key_pos)) { continue; }
|
|
164
214
|
|
|
215
|
+
// Compute Q @ K^T for this position
|
|
165
216
|
var score: f32 = 0.0;
|
|
166
217
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
167
218
|
score = score + q_local[d] * shared_K[k * head_dim + d];
|
|
168
219
|
}
|
|
169
220
|
score = score * scale;
|
|
170
221
|
|
|
222
|
+
// Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
|
|
171
223
|
if (u.attn_softcap > 0.0) {
|
|
172
224
|
score = tanh(score / u.attn_softcap) * u.attn_softcap;
|
|
173
225
|
}
|
|
@@ -176,14 +228,17 @@ fn main(
|
|
|
176
228
|
shared_scores[thread_idx * BLOCK_SIZE + k] = score;
|
|
177
229
|
}
|
|
178
230
|
|
|
231
|
+
// Online softmax update
|
|
179
232
|
let m_new = max(m_i, block_max);
|
|
180
233
|
let correction = exp(m_i - m_new);
|
|
181
234
|
|
|
235
|
+
// Rescale previous accumulator
|
|
182
236
|
l_i = l_i * correction;
|
|
183
237
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
184
238
|
acc[d] = acc[d] * correction;
|
|
185
239
|
}
|
|
186
240
|
|
|
241
|
+
// Add contribution from this block
|
|
187
242
|
for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
|
|
188
243
|
let key_pos = kv_block_start + k;
|
|
189
244
|
if (key_pos >= seq_len) { continue; }
|
|
@@ -193,6 +248,7 @@ fn main(
|
|
|
193
248
|
let p = exp(score - m_new);
|
|
194
249
|
l_i = l_i + p;
|
|
195
250
|
|
|
251
|
+
// Accumulate V contribution
|
|
196
252
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
197
253
|
acc[d] = acc[d] + p * shared_V[k * head_dim + d];
|
|
198
254
|
}
|
|
@@ -204,6 +260,8 @@ fn main(
|
|
|
204
260
|
workgroupBarrier();
|
|
205
261
|
}
|
|
206
262
|
|
|
263
|
+
// Normalize by sum and write output
|
|
264
|
+
// Always write output when valid_query - write zeros if l_i is 0 to avoid garbage
|
|
207
265
|
if (valid_query) {
|
|
208
266
|
let out_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
|
|
209
267
|
let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
|
|
@@ -1,20 +1,23 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
3
|
-
// Fused Multi-Head Attention Kernel
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
|
+
// Fused Multi-Head Attention Kernel
|
|
4
4
|
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
5
|
+
// Implements fused Q @ K^T → scale → mask → softmax → @ V
|
|
6
|
+
// Uses tiled/blocked approach to avoid materializing full attention matrix.
|
|
7
|
+
// Supports grouped query attention (GQA) where numKVHeads < numHeads.
|
|
8
|
+
//
|
|
9
|
+
// Based on Flash Attention principles adapted for WebGPU.
|
|
7
10
|
|
|
8
11
|
enable f16;
|
|
9
12
|
|
|
10
13
|
// Tile sizes for blocked attention
|
|
11
|
-
const MAX_BLOCK_SIZE: u32 =
|
|
14
|
+
const MAX_BLOCK_SIZE: u32 = 32u;
|
|
12
15
|
const MAX_HEAD_TILE: u32 = 64u;
|
|
13
16
|
const MAX_HEAD_DIM: u32 = 64u;
|
|
14
17
|
|
|
15
|
-
override BLOCK_SIZE: u32 =
|
|
18
|
+
override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
|
|
16
19
|
override HEAD_TILE: u32 = 64u; // Head dimension tile
|
|
17
|
-
override WORKGROUP_SIZE: u32 =
|
|
20
|
+
override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
|
|
18
21
|
|
|
19
22
|
struct Uniforms {
|
|
20
23
|
num_heads: u32, // Number of query heads
|
|
@@ -31,7 +34,11 @@ struct Uniforms {
|
|
|
31
34
|
kv_start: u32,
|
|
32
35
|
page_size: u32,
|
|
33
36
|
kv_layout: u32,
|
|
34
|
-
|
|
37
|
+
bidirectional_span_start: u32,
|
|
38
|
+
bidirectional_span_length: u32,
|
|
39
|
+
_pad0: u32,
|
|
40
|
+
_pad1: u32,
|
|
41
|
+
_pad2: u32,
|
|
35
42
|
}
|
|
36
43
|
|
|
37
44
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -54,16 +61,40 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
|
|
|
54
61
|
return query_head_idx / heads_per_kv;
|
|
55
62
|
}
|
|
56
63
|
|
|
64
|
+
fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
|
|
65
|
+
if (u.bidirectional_span_length == 0u) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
let span_start = u.bidirectional_span_start;
|
|
69
|
+
let span_end = span_start + u.bidirectional_span_length;
|
|
70
|
+
return abs_query >= span_start
|
|
71
|
+
&& abs_query < span_end
|
|
72
|
+
&& abs_key >= span_start
|
|
73
|
+
&& abs_key < span_end;
|
|
74
|
+
}
|
|
75
|
+
|
|
57
76
|
// Check if position should be masked (causal + sliding window attention)
|
|
58
77
|
fn is_masked(query_pos: u32, key_pos: u32) -> bool {
|
|
78
|
+
// Compute absolute positions
|
|
59
79
|
let abs_query = query_pos + u.start_pos;
|
|
60
80
|
let abs_key = u.kv_start + key_pos;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
81
|
+
|
|
82
|
+
// Causal mask: query can only attend to keys at same or earlier positions
|
|
83
|
+
if (u.is_causal != 0u && abs_key > abs_query) {
|
|
84
|
+
if (is_bidirectional_span_visible(abs_query, abs_key)) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Sliding window mask: query can only attend to keys within the window
|
|
91
|
+
// Key must be >= (query - window + 1), i.e., within the last `window` positions
|
|
64
92
|
if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
|
|
65
|
-
if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
93
|
+
if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
94
|
+
return true; // Key is too far in the past
|
|
95
|
+
}
|
|
66
96
|
}
|
|
97
|
+
|
|
67
98
|
return false;
|
|
68
99
|
}
|
|
69
100
|
|
|
@@ -93,7 +124,6 @@ fn get_kv_len() -> u32 {
|
|
|
93
124
|
// head_idx and query_block_idx are derived from workgroup_id.x.
|
|
94
125
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
95
126
|
fn main(
|
|
96
|
-
@builtin(global_invocation_id) global_id: vec3<u32>,
|
|
97
127
|
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
98
128
|
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
99
129
|
) {
|
|
@@ -118,7 +148,7 @@ fn main(
|
|
|
118
148
|
// Initialize online softmax accumulators
|
|
119
149
|
var m_i: f32 = -3.402823e+38; // -inf for max tracking
|
|
120
150
|
var l_i: f32 = 0.0; // Sum of exp(x - max)
|
|
121
|
-
var acc: array<f32, MAX_HEAD_DIM>;
|
|
151
|
+
var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
|
|
122
152
|
|
|
123
153
|
// Initialize accumulator
|
|
124
154
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
@@ -189,7 +219,7 @@ fn main(
|
|
|
189
219
|
}
|
|
190
220
|
score = score * scale;
|
|
191
221
|
|
|
192
|
-
// Gemma 2 attention softcapping
|
|
222
|
+
// Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
|
|
193
223
|
if (u.attn_softcap > 0.0) {
|
|
194
224
|
score = tanh(score / u.attn_softcap) * u.attn_softcap;
|
|
195
225
|
}
|