doppler-gpu 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +19 -15
- package/package.json +77 -29
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +4 -2
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +64 -6
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +524 -350
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +12 -2
- package/src/config/schema/doppler.schema.js +108 -8
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +52 -1
- package/src/config/schema/index.js +14 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +68 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +76 -0
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +107 -0
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +0 -31
- package/src/formats/tflite/types.js +5 -5
- package/src/gpu/command-recorder.d.ts +3 -21
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +6 -5
- package/src/gpu/kernels/index.js +3 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +12 -0
- package/src/inference/browser-harness-text-helpers.js +223 -14
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +264 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +17 -0
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +22 -27
- package/src/inference/pipelines/text/embed.js +0 -39
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
- package/src/inference/pipelines/text/generator-helpers.js +4 -5
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +8 -3
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +223 -547
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +5 -2
- package/src/inference/pipelines/text/init.js +183 -724
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.js +2 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +120 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +3 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +223 -27
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +13 -3
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.js +6 -1
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +11 -3
- package/src/storage/artifact-storage-context.js +231 -26
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +44 -67
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +21 -24
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +1 -10
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +2 -0
- package/src/tooling/source-runtime-bundle.js +3 -22
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +1 -7
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/attention_streaming.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Streaming Multi-Head Attention Kernel (f16 KV, no workgroup storage)
|
|
4
4
|
//
|
|
5
5
|
// Same as attention_streaming.wgsl but K/V are stored as f16.
|
|
@@ -24,7 +24,11 @@ struct Uniforms {
|
|
|
24
24
|
kv_start: u32,
|
|
25
25
|
page_size: u32,
|
|
26
26
|
kv_layout: u32,
|
|
27
|
-
|
|
27
|
+
bidirectional_span_start: u32,
|
|
28
|
+
bidirectional_span_length: u32,
|
|
29
|
+
_pad0: u32,
|
|
30
|
+
_pad1: u32,
|
|
31
|
+
_pad2: u32,
|
|
28
32
|
}
|
|
29
33
|
|
|
30
34
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -40,11 +44,26 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
|
|
|
40
44
|
return query_head_idx / heads_per_kv;
|
|
41
45
|
}
|
|
42
46
|
|
|
47
|
+
fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
|
|
48
|
+
if (u.bidirectional_span_length == 0u) {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
let span_start = u.bidirectional_span_start;
|
|
52
|
+
let span_end = span_start + u.bidirectional_span_length;
|
|
53
|
+
return abs_query >= span_start
|
|
54
|
+
&& abs_query < span_end
|
|
55
|
+
&& abs_key >= span_start
|
|
56
|
+
&& abs_key < span_end;
|
|
57
|
+
}
|
|
58
|
+
|
|
43
59
|
fn is_masked(query_pos: u32, key_pos: u32) -> bool {
|
|
44
60
|
let abs_query = query_pos + u.start_pos;
|
|
45
61
|
let abs_key = u.kv_start + key_pos;
|
|
46
62
|
// Causal mask
|
|
47
|
-
if (u.is_causal != 0u && abs_key > abs_query) {
|
|
63
|
+
if (u.is_causal != 0u && abs_key > abs_query) {
|
|
64
|
+
if (is_bidirectional_span_visible(abs_query, abs_key)) { return false; }
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
48
67
|
// Sliding window mask
|
|
49
68
|
if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
|
|
50
69
|
if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
|
|
@@ -2,16 +2,22 @@ export { runEmbedBackward, recordEmbedBackward } from './embed_backward.js';
|
|
|
2
2
|
export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
|
|
3
3
|
export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
|
|
4
4
|
export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
|
|
5
|
+
export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
|
|
5
6
|
export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
|
|
7
|
+
export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
|
|
6
8
|
export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
|
|
7
9
|
export { runRoPEBackward, recordRoPEBackward } from './rope_backward.js';
|
|
10
|
+
export { runRoPEBackward as runRopeBackward, recordRoPEBackward as recordRopeBackward } from './rope_backward.js';
|
|
8
11
|
export { runSiluBackward, recordSiluBackward } from './silu_backward.js';
|
|
9
12
|
export { runGeluBackward, recordGeluBackward } from './gelu_backward.js';
|
|
10
13
|
export { runScaleBackward, recordScaleBackward } from './scale_backward.js';
|
|
11
14
|
export { runCrossEntropyBackward, recordCrossEntropyBackward } from './cross_entropy_backward.js';
|
|
12
15
|
export { runBiasAddBackward, recordBiasAddBackward } from './bias_add_backward.js';
|
|
13
16
|
export { runUpsample2DBackward, recordUpsample2DBackward } from './upsample2d_backward.js';
|
|
17
|
+
export { runUpsample2DBackward as runUpsample2dBackward, recordUpsample2DBackward as recordUpsample2dBackward } from './upsample2d_backward.js';
|
|
14
18
|
export { runPixelShuffleBackward, recordPixelShuffleBackward } from './pixel_shuffle_backward.js';
|
|
15
19
|
export { runGroupNormBackward, recordGroupNormBackward } from './groupnorm_backward.js';
|
|
20
|
+
export { runGroupNormBackward as runGroupnormBackward, recordGroupNormBackward as recordGroupnormBackward } from './groupnorm_backward.js';
|
|
16
21
|
export { runConv2DBackward, recordConv2DBackward } from './conv2d_backward.js';
|
|
22
|
+
export { runConv2DBackward as runConv2dBackward, recordConv2DBackward as recordConv2dBackward } from './conv2d_backward.js';
|
|
17
23
|
export { runAdam, recordAdam } from './adam.js';
|
|
@@ -27,7 +27,6 @@
|
|
|
27
27
|
|
|
28
28
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
29
29
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
30
|
-
override PARAMS_IS_F16: bool = false; // Weight/bias packed as f16 pairs (unused for now as we read f32)
|
|
31
30
|
|
|
32
31
|
// =============================================================================
|
|
33
32
|
// Uniforms
|
|
@@ -43,3 +43,38 @@ export declare function recordMatmulTransposeA(
|
|
|
43
43
|
K: number,
|
|
44
44
|
options?: { alpha?: number; outputBuffer?: GPUBuffer | null }
|
|
45
45
|
): Promise<Tensor>;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Factory that builds a backward-kernel runner pair from an op name
|
|
49
|
+
* and a small spec (uniform-writer + workgroup sizing).
|
|
50
|
+
*/
|
|
51
|
+
export declare function createBackwardKernel(
|
|
52
|
+
opName: string,
|
|
53
|
+
spec: Record<string, unknown>
|
|
54
|
+
): {
|
|
55
|
+
run: (...args: unknown[]) => Promise<Tensor>;
|
|
56
|
+
record: (recorder: CommandRecorder, ...args: unknown[]) => Promise<Tensor>;
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Matmul-backward gradient w.r.t. input (dX = dY · Wᵀ). Submit + wait.
|
|
61
|
+
*/
|
|
62
|
+
export declare function runMatmulBackwardDx(
|
|
63
|
+
dY: Tensor,
|
|
64
|
+
W: Tensor,
|
|
65
|
+
M: number,
|
|
66
|
+
K: number,
|
|
67
|
+
N: number,
|
|
68
|
+
options?: { outputBuffer?: GPUBuffer | null }
|
|
69
|
+
): Promise<Tensor>;
|
|
70
|
+
|
|
71
|
+
/** Record runMatmulBackwardDx onto a CommandRecorder. */
|
|
72
|
+
export declare function recordMatmulBackwardDx(
|
|
73
|
+
recorder: CommandRecorder,
|
|
74
|
+
dY: Tensor,
|
|
75
|
+
W: Tensor,
|
|
76
|
+
M: number,
|
|
77
|
+
K: number,
|
|
78
|
+
N: number,
|
|
79
|
+
options?: { outputBuffer?: GPUBuffer | null }
|
|
80
|
+
): Promise<Tensor>;
|
|
@@ -52,19 +52,3 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
|
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
// Single-element version for odd-sized tensors
|
|
56
|
-
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
57
|
-
fn main_single(@builtin(global_invocation_id) global_id: vec3<u32>) {
|
|
58
|
-
let idx = global_id.x;
|
|
59
|
-
|
|
60
|
-
if (idx >= u.num_elements) {
|
|
61
|
-
return;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// Read as u16 pairs, extract the right one
|
|
65
|
-
let pair_idx = idx / 2u;
|
|
66
|
-
let packed = input[pair_idx];
|
|
67
|
-
|
|
68
|
-
let bf16 = select(packed & 0xFFFFu, (packed >> 16u) & 0xFFFFu, (idx & 1u) == 1u);
|
|
69
|
-
output[idx] = bitcast<f32>(bf16 << 16u);
|
|
70
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/bias_add.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// bias_add_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,59 +1,15 @@
|
|
|
1
1
|
--- src/gpu/kernels/attention.wgsl
|
|
2
2
|
+++ src/gpu/kernels/attention_f16.wgsl
|
|
3
|
-
@@ -
|
|
4
|
-
-// Fused Multi-Head Attention Kernel
|
|
5
|
-
+// Fused Multi-Head Attention Kernel (f16 QKV + f16 output)
|
|
3
|
+
@@ -6,6 +6,8 @@
|
|
6
4
|
//
|
|
7
|
-
|
|
8
|
-
-// Uses tiled/blocked approach to avoid materializing full attention matrix.
|
|
9
|
-
-// Supports grouped query attention (GQA) where numKVHeads < numHeads.
|
|
10
|
-
-//
|
|
11
|
-
-// Based on Flash Attention principles adapted for WebGPU.
|
|
12
|
-
+// Same algorithm as attention.wgsl but Q/K/V are f16 and output is f16.
|
|
13
|
-
+// Computation is performed in f32 for stability.
|
|
14
|
-
+
|
|
15
|
-
+enable f16;
|
|
5
|
+
// Based on Flash Attention principles adapted for WebGPU.
|
|
16
6
|
|
|
7
|
+
+enable f16;
|
|
8
|
+
+
|
|
17
9
|
// Tile sizes for blocked attention
|
|
18
|
-
|
|
19
|
-
+const MAX_BLOCK_SIZE: u32 = 64u;
|
|
10
|
+
const MAX_BLOCK_SIZE: u32 = 32u;
|
|
20
11
|
const MAX_HEAD_TILE: u32 = 64u;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
-override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
|
|
24
|
-
-override HEAD_TILE: u32 = 64u; // Head dimension tile
|
|
25
|
-
-override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
|
|
26
|
-
+override BLOCK_SIZE: u32 = 64u;
|
|
27
|
-
+override HEAD_TILE: u32 = 64u;
|
|
28
|
-
+override WORKGROUP_SIZE: u32 = 64u;
|
|
29
|
-
|
|
30
|
-
struct Uniforms {
|
|
31
|
-
- num_heads: u32, // Number of query heads
|
|
32
|
-
- num_kv_heads: u32, // Number of KV heads (for GQA)
|
|
33
|
-
- head_dim: u32, // Dimension per head
|
|
34
|
-
- seq_len: u32, // Current sequence length (for KV)
|
|
35
|
-
- query_len: u32, // Query length (1 for decode, seq_len for prefill)
|
|
36
|
-
- scale: f32, // 1/sqrt(head_dim)
|
|
37
|
-
- is_causal: u32, // Apply causal mask (1 = yes)
|
|
38
|
-
- start_pos: u32, // Absolute position offset for causal masking
|
|
39
|
-
- attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
|
|
40
|
-
- sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
|
|
41
|
-
- kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
|
|
42
|
-
+ num_heads: u32,
|
|
43
|
-
+ num_kv_heads: u32,
|
|
44
|
-
+ head_dim: u32,
|
|
45
|
-
+ seq_len: u32,
|
|
46
|
-
+ query_len: u32,
|
|
47
|
-
+ scale: f32,
|
|
48
|
-
+ is_causal: u32,
|
|
49
|
-
+ start_pos: u32,
|
|
50
|
-
+ attn_softcap: f32,
|
|
51
|
-
+ sliding_window: u32,
|
|
52
|
-
+ kv_len_source: u32,
|
|
53
|
-
kv_start: u32,
|
|
54
|
-
page_size: u32,
|
|
55
|
-
kv_layout: u32,
|
|
56
|
-
@@ -34,44 +33,29 @@
|
|
12
|
+
@@ -38,10 +40,10 @@
|
|
57
13
|
}
|
|
58
14
|
|
|
59
15
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
@@ -61,107 +17,23 @@
|
|
|
61
17
|
-@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
|
|
62
18
|
-@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
|
|
63
19
|
-@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
|
|
64
|
-
+@group(0) @binding(1) var<storage, read> Q: array<f16>;
|
|
65
|
-
+@group(0) @binding(2) var<storage, read> K: array<f16>;
|
|
66
|
-
+@group(0) @binding(3) var<storage, read> V: array<f16>;
|
|
67
|
-
+@group(0) @binding(4) var<storage, read_write> output: array<f16>;
|
|
20
|
+
+@group(0) @binding(1) var<storage, read> Q: array<f16>; // [query_len, num_heads, head_dim]
|
|
21
|
+
+@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
|
|
22
|
+
+@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
|
|
23
|
+
+@group(0) @binding(4) var<storage, read_write> output: array<f16>; // [query_len, num_heads, head_dim]
|
|
68
24
|
@group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
|
|
69
25
|
@group(0) @binding(6) var<storage, read> page_table: array<u32>;
|
|
70
26
|
|
|
71
|
-
|
|
72
|
-
var<workgroup> shared_K: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
|
|
73
|
-
var<workgroup> shared_V: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
|
|
74
|
-
var<workgroup> shared_scores: array<f32, MAX_BLOCK_SIZE * MAX_BLOCK_SIZE>;
|
|
75
|
-
|
|
76
|
-
-// Get KV head index for grouped query attention
|
|
77
|
-
fn get_kv_head_idx(query_head_idx: u32) -> u32 {
|
|
78
|
-
- // GQA: multiple query heads share one KV head
|
|
79
|
-
let heads_per_kv = u.num_heads / u.num_kv_heads;
|
|
80
|
-
return query_head_idx / heads_per_kv;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
-// Check if position should be masked (causal + sliding window attention)
|
|
84
|
-
fn is_masked(query_pos: u32, key_pos: u32) -> bool {
|
|
85
|
-
- // Compute absolute positions
|
|
86
|
-
let abs_query = query_pos + u.start_pos;
|
|
87
|
-
let abs_key = u.kv_start + key_pos;
|
|
88
|
-
-
|
|
89
|
-
- // Causal mask: query can only attend to keys at same or earlier positions
|
|
90
|
-
- if (u.is_causal != 0u && abs_key > abs_query) {
|
|
91
|
-
- return true;
|
|
92
|
-
- }
|
|
93
|
-
-
|
|
94
|
-
- // Sliding window mask: query can only attend to keys within the window
|
|
95
|
-
- // Key must be >= (query - window + 1), i.e., within the last `window` positions
|
|
96
|
-
+ if (u.is_causal != 0u && abs_key > abs_query) { return true; }
|
|
97
|
-
if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
|
|
98
|
-
- if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
99
|
-
- return true; // Key is too far in the past
|
|
100
|
-
- }
|
|
101
|
-
- }
|
|
102
|
-
-
|
|
103
|
-
+ if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
|
|
104
|
-
+ }
|
|
105
|
-
return false;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
@@ -96,11 +80,9 @@
|
|
109
|
-
return kv_len_buffer[0];
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
-// Main attention kernel - one workgroup per (query_block, head)
|
|
113
|
-
-// Workgroups are dispatched linearly as: numQueryBlocks * numHeads.
|
|
114
|
-
-// head_idx and query_block_idx are derived from workgroup_id.x.
|
|
115
|
-
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
116
|
-
fn main(
|
|
117
|
-
+ @builtin(global_invocation_id) global_id: vec3<u32>,
|
|
118
|
-
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
119
|
-
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
120
|
-
) {
|
|
121
|
-
@@ -118,42 +100,36 @@
|
|
122
|
-
let query_len = u.query_len;
|
|
123
|
-
let scale = u.scale;
|
|
124
|
-
|
|
125
|
-
- // Query position this thread handles
|
|
126
|
-
let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
|
|
127
|
-
let valid_query = query_pos < query_len;
|
|
128
|
-
|
|
129
|
-
- // Initialize online softmax accumulators
|
|
130
|
-
- var m_i: f32 = -3.402823e+38; // -inf for max tracking
|
|
131
|
-
- var l_i: f32 = 0.0; // Sum of exp(x - max)
|
|
132
|
-
- var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
|
|
133
|
-
-
|
|
134
|
-
- // Initialize accumulator
|
|
135
|
-
+ var m_i: f32 = -3.402823e+38;
|
|
136
|
-
+ var l_i: f32 = 0.0;
|
|
137
|
-
+ var acc: array<f32, MAX_HEAD_DIM>;
|
|
138
|
-
+
|
|
139
|
-
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
140
|
-
acc[d] = 0.0;
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
- // Load query for this thread into registers
|
|
144
|
-
var q_local: array<f32, MAX_HEAD_DIM>;
|
|
27
|
+
@@ -156,7 +158,7 @@
|
|
145
28
|
if (valid_query) {
|
|
146
29
|
let q_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
|
|
147
30
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
148
31
|
- q_local[d] = Q[q_offset + d];
|
|
149
|
-
- }
|
|
150
|
-
- }
|
|
151
|
-
-
|
|
152
|
-
- // Process key-value blocks
|
|
153
32
|
+ q_local[d] = f32(Q[q_offset + d]);
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
+
|
|
157
|
-
let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
|
|
158
|
-
|
|
159
|
-
for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
|
|
160
|
-
let kv_block_start = kv_block * BLOCK_SIZE;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
161
35
|
|
|
162
|
-
-
|
|
163
|
-
let k_load_idx = kv_block_start + thread_idx;
|
|
164
|
-
if (k_load_idx < seq_len) {
|
|
36
|
+
@@ -172,7 +174,7 @@
|
|
165
37
|
let k_idx = get_kv_pos(k_load_idx);
|
|
166
38
|
let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
167
39
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
@@ -170,13 +42,7 @@
|
|
|
170
42
|
}
|
|
171
43
|
} else {
|
|
172
44
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
173
|
-
@@ -
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
- // Load V block
|
|
178
|
-
let v_load_idx = kv_block_start + thread_idx;
|
|
179
|
-
if (v_load_idx < seq_len) {
|
|
45
|
+
@@ -186,7 +188,7 @@
|
|
180
46
|
let v_idx = get_kv_pos(v_load_idx);
|
|
181
47
|
let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
182
48
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
@@ -185,74 +51,12 @@
|
|
|
185
51
|
}
|
|
186
52
|
} else {
|
|
187
53
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
188
|
-
@@ -
|
|
189
|
-
|
|
190
|
-
workgroupBarrier();
|
|
191
|
-
|
|
192
|
-
- // Compute attention scores for this block
|
|
193
|
-
if (valid_query) {
|
|
194
|
-
- // Find max in this block (for numerical stability)
|
|
195
|
-
var block_max: f32 = -3.402823e+38;
|
|
196
|
-
|
|
197
|
-
for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
|
|
198
|
-
let key_pos = kv_block_start + k;
|
|
199
|
-
if (key_pos >= seq_len) { continue; }
|
|
200
|
-
-
|
|
201
|
-
- // Check causal mask
|
|
202
|
-
if (is_masked(query_pos, key_pos)) { continue; }
|
|
203
|
-
|
|
204
|
-
- // Compute Q @ K^T for this position
|
|
205
|
-
var score: f32 = 0.0;
|
|
206
|
-
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
207
|
-
score = score + q_local[d] * shared_K[k * head_dim + d];
|
|
208
|
-
}
|
|
209
|
-
score = score * scale;
|
|
210
|
-
|
|
211
|
-
- // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
|
|
212
|
-
if (u.attn_softcap > 0.0) {
|
|
213
|
-
score = tanh(score / u.attn_softcap) * u.attn_softcap;
|
|
214
|
-
}
|
|
215
|
-
@@ -205,17 +174,14 @@
|
|
216
|
-
shared_scores[thread_idx * BLOCK_SIZE + k] = score;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
- // Online softmax update
|
|
220
|
-
let m_new = max(m_i, block_max);
|
|
221
|
-
let correction = exp(m_i - m_new);
|
|
222
|
-
|
|
223
|
-
- // Rescale previous accumulator
|
|
224
|
-
l_i = l_i * correction;
|
|
225
|
-
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
226
|
-
acc[d] = acc[d] * correction;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
- // Add contribution from this block
|
|
230
|
-
for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
|
|
231
|
-
let key_pos = kv_block_start + k;
|
|
232
|
-
if (key_pos >= seq_len) { continue; }
|
|
233
|
-
@@ -225,7 +191,6 @@
|
|
234
|
-
let p = exp(score - m_new);
|
|
235
|
-
l_i = l_i + p;
|
|
236
|
-
|
|
237
|
-
- // Accumulate V contribution
|
|
238
|
-
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
239
|
-
acc[d] = acc[d] + p * shared_V[k * head_dim + d];
|
|
240
|
-
}
|
|
241
|
-
@@ -237,13 +202,11 @@
|
|
242
|
-
workgroupBarrier();
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
- // Normalize by sum and write output
|
|
246
|
-
- // Always write output when valid_query - write zeros if l_i is 0 to avoid garbage
|
|
247
|
-
if (valid_query) {
|
|
54
|
+
@@ -262,7 +264,7 @@
|
|
248
55
|
let out_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
|
|
249
56
|
let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
|
|
250
57
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
251
58
|
- output[out_offset + d] = acc[d] * inv_l_i;
|
|
252
|
-
- }
|
|
253
|
-
- }
|
|
254
|
-
-}
|
|
255
59
|
+ output[out_offset + d] = f16(acc[d] * inv_l_i);
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -1,34 +1,15 @@
|
|
|
1
1
|
--- src/gpu/kernels/attention.wgsl
|
|
2
2
|
+++ src/gpu/kernels/attention_f16kv.wgsl
|
|
3
|
-
@@ -
|
|
4
|
-
-// Fused Multi-Head Attention Kernel
|
|
5
|
-
+// Fused Multi-Head Attention Kernel (f16 KV)
|
|
3
|
+
@@ -6,6 +6,8 @@
|
|
6
4
|
//
|
|
7
|
-
|
|
8
|
-
-// Uses tiled/blocked approach to avoid materializing full attention matrix.
|
|
9
|
-
-// Supports grouped query attention (GQA) where numKVHeads < numHeads.
|
|
10
|
-
-//
|
|
11
|
-
-// Based on Flash Attention principles adapted for WebGPU.
|
|
12
|
-
+// Same as attention.wgsl but K/V are stored as f16 for KV-cache compression.
|
|
13
|
-
+// Q and output remain f32; K/V are cast to f32 on load.
|
|
14
|
-
+
|
|
15
|
-
+enable f16;
|
|
5
|
+
// Based on Flash Attention principles adapted for WebGPU.
|
|
16
6
|
|
|
7
|
+
+enable f16;
|
|
8
|
+
+
|
|
17
9
|
// Tile sizes for blocked attention
|
|
18
|
-
|
|
19
|
-
+const MAX_BLOCK_SIZE: u32 = 64u;
|
|
10
|
+
const MAX_BLOCK_SIZE: u32 = 32u;
|
|
20
11
|
const MAX_HEAD_TILE: u32 = 64u;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
-override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
|
|
24
|
-
+override BLOCK_SIZE: u32 = 64u; // Sequence tile size (must match WORKGROUP_SIZE)
|
|
25
|
-
override HEAD_TILE: u32 = 64u; // Head dimension tile
|
|
26
|
-
-override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
|
|
27
|
-
+override WORKGROUP_SIZE: u32 = 64u; // One thread per query position
|
|
28
|
-
|
|
29
|
-
struct Uniforms {
|
|
30
|
-
num_heads: u32, // Number of query heads
|
|
31
|
-
@@ -35,8 +34,8 @@
|
|
12
|
+
@@ -39,8 +41,8 @@
|
|
32
13
|
|
|
33
14
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
34
15
|
@group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
|
|
@@ -39,53 +20,7 @@
|
|
|
39
20
|
@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
|
|
40
21
|
@group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
|
|
41
22
|
@group(0) @binding(6) var<storage, read> page_table: array<u32>;
|
|
42
|
-
@@ -
|
|
43
|
-
|
|
44
|
-
// Check if position should be masked (causal + sliding window attention)
|
|
45
|
-
fn is_masked(query_pos: u32, key_pos: u32) -> bool {
|
|
46
|
-
- // Compute absolute positions
|
|
47
|
-
let abs_query = query_pos + u.start_pos;
|
|
48
|
-
let abs_key = u.kv_start + key_pos;
|
|
49
|
-
-
|
|
50
|
-
- // Causal mask: query can only attend to keys at same or earlier positions
|
|
51
|
-
- if (u.is_causal != 0u && abs_key > abs_query) {
|
|
52
|
-
- return true;
|
|
53
|
-
- }
|
|
54
|
-
-
|
|
55
|
-
- // Sliding window mask: query can only attend to keys within the window
|
|
56
|
-
- // Key must be >= (query - window + 1), i.e., within the last `window` positions
|
|
57
|
-
+ // Causal mask
|
|
58
|
-
+ if (u.is_causal != 0u && abs_key > abs_query) { return true; }
|
|
59
|
-
+ // Sliding window mask
|
|
60
|
-
if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
|
|
61
|
-
- if (abs_key < abs_query - u.sliding_window + 1u) {
|
|
62
|
-
- return true; // Key is too far in the past
|
|
63
|
-
- }
|
|
64
|
-
- }
|
|
65
|
-
-
|
|
66
|
-
+ if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
|
|
67
|
-
+ }
|
|
68
|
-
return false;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
@@ -101,6 +91,7 @@
|
|
72
|
-
// head_idx and query_block_idx are derived from workgroup_id.x.
|
|
73
|
-
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
74
|
-
fn main(
|
|
75
|
-
+ @builtin(global_invocation_id) global_id: vec3<u32>,
|
|
76
|
-
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
77
|
-
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
78
|
-
) {
|
|
79
|
-
@@ -125,7 +116,7 @@
|
|
80
|
-
// Initialize online softmax accumulators
|
|
81
|
-
var m_i: f32 = -3.402823e+38; // -inf for max tracking
|
|
82
|
-
var l_i: f32 = 0.0; // Sum of exp(x - max)
|
|
83
|
-
- var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
|
|
84
|
-
+ var acc: array<f32, MAX_HEAD_DIM>;
|
|
85
|
-
|
|
86
|
-
// Initialize accumulator
|
|
87
|
-
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
88
|
-
@@ -153,7 +144,7 @@
|
|
23
|
+
@@ -172,7 +174,7 @@
|
|
89
24
|
let k_idx = get_kv_pos(k_load_idx);
|
|
90
25
|
let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
91
26
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
@@ -94,7 +29,7 @@
|
|
|
94
29
|
}
|
|
95
30
|
} else {
|
|
96
31
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
97
|
-
@@ -
|
|
32
|
+
@@ -186,7 +188,7 @@
|
|
98
33
|
let v_idx = get_kv_pos(v_load_idx);
|
|
99
34
|
let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
|
|
100
35
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
@@ -103,12 +38,3 @@
|
|
|
103
38
|
}
|
|
104
39
|
} else {
|
|
105
40
|
for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
|
|
106
|
-
@@ -196,7 +187,7 @@
|
|
107
|
-
}
|
|
108
|
-
score = score * scale;
|
|
109
|
-
|
|
110
|
-
- // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
|
|
111
|
-
+ // Gemma 2 attention softcapping
|
|
112
|
-
if (u.attn_softcap > 0.0) {
|
|
113
|
-
score = tanh(score / u.attn_softcap) * u.attn_softcap;
|
|
114
|
-
}
|