doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -8,6 +8,7 @@ enable subgroups;
8
8
  const MAX_WORKGROUP_SIZE: u32 = 256u;
9
9
  const MAX_SUBGROUPS: u32 = 256u;
10
10
  const MAX_HEAD_DIM: u32 = 512u;
11
+ const NEG_INF = f16(-65504.0);
11
12
 
12
13
  override WORKGROUP_SIZE: u32 = 256u;
13
14
 
@@ -37,12 +38,12 @@ struct Uniforms {
37
38
  @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
38
39
  @group(0) @binding(6) var<storage, read> page_table: array<u32>;
39
40
 
40
- var<workgroup> shared_q: array<f32, MAX_HEAD_DIM>;
41
- var<workgroup> shared_scores: array<f32, MAX_WORKGROUP_SIZE>;
42
- var<workgroup> sg_max: array<f32, MAX_SUBGROUPS>;
43
- var<workgroup> sg_sum: array<f32, MAX_SUBGROUPS>;
44
- var<workgroup> global_max: f32;
45
- var<workgroup> global_sum: f32;
41
+ var<workgroup> shared_q: array<f16, MAX_HEAD_DIM>;
42
+ var<workgroup> shared_scores: array<f16, MAX_WORKGROUP_SIZE>;
43
+ var<workgroup> sg_max: array<f16, MAX_SUBGROUPS>;
44
+ var<workgroup> sg_sum: array<f16, MAX_SUBGROUPS>;
45
+ var<workgroup> global_max: f16;
46
+ var<workgroup> global_sum: f16;
46
47
 
47
48
  fn get_kv_head_idx(query_head_idx: u32) -> u32 {
48
49
  let heads_per_kv = u.num_heads / u.num_kv_heads;
@@ -90,6 +91,8 @@ fn main(
90
91
  let tid = local_id.x;
91
92
  let head_dim = u.head_dim;
92
93
  let kv_len = get_kv_len();
94
+ let scale = f16(u.scale);
95
+ let softcap = f16(u.attn_softcap);
93
96
  let q_offset = head_idx * head_dim;
94
97
  let out_dim0 = tid;
95
98
  let out_dim1 = tid + WORKGROUP_SIZE;
@@ -119,17 +122,17 @@ fn main(
119
122
  let kv_head_idx = get_kv_head_idx(head_idx);
120
123
 
121
124
  if (has_out_dim0) {
122
- shared_q[out_dim0] = f32(Q[q_offset + out_dim0]);
125
+ shared_q[out_dim0] = Q[q_offset + out_dim0];
123
126
  }
124
127
  if (has_out_dim1) {
125
- shared_q[out_dim1] = f32(Q[q_offset + out_dim1]);
128
+ shared_q[out_dim1] = Q[q_offset + out_dim1];
126
129
  }
127
130
  workgroupBarrier();
128
131
 
129
- var running_max: f32 = -3.402823e+38;
130
- var running_sum: f32 = 0.0;
131
- var out_accum0: f32 = 0.0;
132
- var out_accum1: f32 = 0.0;
132
+ var running_max: f16 = NEG_INF;
133
+ var running_sum: f16 = f16(0.0);
134
+ var out_accum0: f16 = f16(0.0);
135
+ var out_accum1: f16 = f16(0.0);
133
136
 
134
137
  var start_k: u32 = 0u;
135
138
  if (u.sliding_window > 0u && kv_len > u.sliding_window) {
@@ -142,7 +145,7 @@ fn main(
142
145
  let k_pos = k_start + tid;
143
146
  let valid_k = k_pos < kv_len;
144
147
  var masked = false;
145
- var score: f32 = -3.402823e+38;
148
+ var score: f16 = NEG_INF;
146
149
 
147
150
  if (valid_k) {
148
151
  let abs_key = u.kv_start + k_pos;
@@ -150,32 +153,32 @@ fn main(
150
153
  if (!masked) {
151
154
  let k_idx = get_kv_pos(k_pos);
152
155
  let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
153
- var dot: f32 = 0.0;
156
+ var dot: f16 = f16(0.0);
154
157
  for (var d: u32 = 0u; d < head_dim; d = d + 2u) {
155
158
  let q0 = shared_q[d];
156
- let k0 = f32(K[k_offset + d]);
159
+ let k0 = K[k_offset + d];
157
160
  dot = dot + q0 * k0;
158
161
  if (d + 1u < head_dim) {
159
162
  let q1 = shared_q[d + 1u];
160
- let k1 = f32(K[k_offset + d + 1u]);
163
+ let k1 = K[k_offset + d + 1u];
161
164
  dot = dot + q1 * k1;
162
165
  }
163
166
  }
164
- score = dot * u.scale;
165
- if (u.attn_softcap > 0.0) {
166
- score = tanh(score / u.attn_softcap) * u.attn_softcap;
167
+ score = dot * scale;
168
+ if (softcap > f16(0.0)) {
169
+ score = tanh(score / softcap) * softcap;
167
170
  }
168
171
  }
169
172
  }
170
173
 
171
- var chunk_max = subgroupMax(score);
174
+ let chunk_max = subgroupMax(score);
172
175
  if (sg_tid == 0u && subgroup_id < num_subgroups) {
173
176
  sg_max[subgroup_id] = chunk_max;
174
177
  }
175
178
  workgroupBarrier();
176
179
 
177
180
  if (tid == 0u) {
178
- var m: f32 = -3.402823e+38;
181
+ var m: f16 = NEG_INF;
179
182
  for (var s: u32 = 0u; s < num_subgroups; s++) {
180
183
  m = max(m, sg_max[s]);
181
184
  }
@@ -187,20 +190,20 @@ fn main(
187
190
  let new_max = max(running_max, chunk_max_val);
188
191
  let rescale = exp(running_max - new_max);
189
192
 
190
- var exp_score: f32 = 0.0;
193
+ var exp_score: f16 = f16(0.0);
191
194
  if (valid_k && !masked) {
192
195
  exp_score = exp(score - new_max);
193
196
  }
194
197
  shared_scores[tid] = exp_score;
195
198
 
196
- var chunk_sum = subgroupAdd(exp_score);
199
+ let chunk_sum = subgroupAdd(exp_score);
197
200
  if (sg_tid == 0u && subgroup_id < num_subgroups) {
198
201
  sg_sum[subgroup_id] = chunk_sum;
199
202
  }
200
203
  workgroupBarrier();
201
204
 
202
205
  if (tid == 0u) {
203
- var s: f32 = 0.0;
206
+ var s: f16 = f16(0.0);
204
207
  for (var i: u32 = 0u; i < num_subgroups; i++) {
205
208
  s = s + sg_sum[i];
206
209
  }
@@ -221,21 +224,21 @@ fn main(
221
224
  let v_idx = get_kv_pos(k_pos_inner);
222
225
  let v_base = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
223
226
  if (has_out_dim0) {
224
- out_accum0 = out_accum0 + shared_scores[score_idx] * f32(V[v_base + out_dim0]);
227
+ out_accum0 = out_accum0 + shared_scores[score_idx] * V[v_base + out_dim0];
225
228
  }
226
229
  if (has_out_dim1) {
227
- out_accum1 = out_accum1 + shared_scores[score_idx] * f32(V[v_base + out_dim1]);
230
+ out_accum1 = out_accum1 + shared_scores[score_idx] * V[v_base + out_dim1];
228
231
  }
229
232
  }
230
233
  }
231
234
  workgroupBarrier();
232
235
  }
233
236
 
234
- let inv_sum = select(0.0, 1.0 / running_sum, running_sum > 0.0);
237
+ let inv_sum = select(f16(0.0), f16(1.0) / running_sum, running_sum > f16(0.0));
235
238
  if (has_out_dim0) {
236
- output[q_offset + out_dim0] = f16(out_accum0 * inv_sum);
239
+ output[q_offset + out_dim0] = out_accum0 * inv_sum;
237
240
  }
238
241
  if (has_out_dim1) {
239
- output[q_offset + out_dim1] = f16(out_accum1 * inv_sum);
242
+ output[q_offset + out_dim1] = out_accum1 * inv_sum;
240
243
  }
241
244
  }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_decode_online_f16.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Online Decode Attention Kernel (f32 Q + f16 KV + f32 output)
4
4
  //
5
5
  // Uses online softmax with subgroup reductions and chunked KV processing.
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_decode_paged_f16.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Paged Decode Attention Kernel (f16 KV)
4
4
  //
5
5
  // Q is f32, K/V are f16, output is f32. Uses page table indirection.
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_decode_tiered_f16.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Tiered Decode Attention Kernel (f32 Q + f16 KV)
4
4
  //
5
5
  // Hot ring buffer + cold paged cache with split-domain softmax.
@@ -14,7 +14,6 @@ const MAX_HEAD_DIM: u32 = 256u;
14
14
  override WORKGROUP_SIZE: u32 = 256u;
15
15
  override BIT_WIDTH: u32 = 4u;
16
16
  override PACK_FACTOR: u32 = 8u;
17
- override NUM_CENTROIDS: u32 = 16u;
18
17
 
19
18
  struct Uniforms {
20
19
  num_heads: u32,
@@ -14,7 +14,6 @@ const MAX_HEAD_DIM: u32 = 256u;
14
14
  override WORKGROUP_SIZE: u32 = 256u;
15
15
  override BIT_WIDTH_MSE: u32 = 3u;
16
16
  override PACK_FACTOR_MSE: u32 = 10u;
17
- override NUM_CENTROIDS_MSE: u32 = 8u;
18
17
 
19
18
  struct Uniforms {
20
19
  num_heads: u32,
@@ -1,63 +1,100 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
3
- // Fused Multi-Head Attention Kernel (f16 QKV + f16 output)
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
+ // Fused Multi-Head Attention Kernel
4
4
  //
5
- // Same algorithm as attention.wgsl but Q/K/V are f16 and output is f16.
6
- // Computation is performed in f32 for stability.
5
+ // Implements fused Q @ K^T → scale → mask → softmax → @ V
6
+ // Uses tiled/blocked approach to avoid materializing full attention matrix.
7
+ // Supports grouped query attention (GQA) where numKVHeads < numHeads.
8
+ //
9
+ // Based on Flash Attention principles adapted for WebGPU.
7
10
 
8
11
  enable f16;
9
12
 
10
13
  // Tile sizes for blocked attention
11
- const MAX_BLOCK_SIZE: u32 = 64u;
14
+ const MAX_BLOCK_SIZE: u32 = 32u;
12
15
  const MAX_HEAD_TILE: u32 = 64u;
13
16
  const MAX_HEAD_DIM: u32 = 64u;
14
17
 
15
- override BLOCK_SIZE: u32 = 64u;
16
- override HEAD_TILE: u32 = 64u;
17
- override WORKGROUP_SIZE: u32 = 64u;
18
+ override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
19
+ override HEAD_TILE: u32 = 64u; // Head dimension tile
20
+ override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
18
21
 
19
22
  struct Uniforms {
20
- num_heads: u32,
21
- num_kv_heads: u32,
22
- head_dim: u32,
23
- seq_len: u32,
24
- query_len: u32,
25
- scale: f32,
26
- is_causal: u32,
27
- start_pos: u32,
28
- attn_softcap: f32,
29
- sliding_window: u32,
30
- kv_len_source: u32,
23
+ num_heads: u32, // Number of query heads
24
+ num_kv_heads: u32, // Number of KV heads (for GQA)
25
+ head_dim: u32, // Dimension per head
26
+ seq_len: u32, // Current sequence length (for KV)
27
+ query_len: u32, // Query length (1 for decode, seq_len for prefill)
28
+ scale: f32, // 1/sqrt(head_dim)
29
+ is_causal: u32, // Apply causal mask (1 = yes)
30
+ start_pos: u32, // Absolute position offset for causal masking
31
+ attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
32
+ sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
33
+ kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
31
34
  kv_start: u32,
32
35
  page_size: u32,
33
36
  kv_layout: u32,
34
- _pad: u32,
37
+ bidirectional_span_start: u32,
38
+ bidirectional_span_length: u32,
39
+ _pad0: u32,
40
+ _pad1: u32,
41
+ _pad2: u32,
35
42
  }
36
43
 
37
44
  @group(0) @binding(0) var<uniform> u: Uniforms;
38
- @group(0) @binding(1) var<storage, read> Q: array<f16>;
39
- @group(0) @binding(2) var<storage, read> K: array<f16>;
40
- @group(0) @binding(3) var<storage, read> V: array<f16>;
41
- @group(0) @binding(4) var<storage, read_write> output: array<f16>;
45
+ @group(0) @binding(1) var<storage, read> Q: array<f16>; // [query_len, num_heads, head_dim]
46
+ @group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
47
+ @group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
48
+ @group(0) @binding(4) var<storage, read_write> output: array<f16>; // [query_len, num_heads, head_dim]
42
49
  @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
43
50
  @group(0) @binding(6) var<storage, read> page_table: array<u32>;
44
51
 
52
+ // Shared memory for tiled computation
45
53
  var<workgroup> shared_K: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
46
54
  var<workgroup> shared_V: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
47
55
  var<workgroup> shared_scores: array<f32, MAX_BLOCK_SIZE * MAX_BLOCK_SIZE>;
48
56
 
57
+ // Get KV head index for grouped query attention
49
58
  fn get_kv_head_idx(query_head_idx: u32) -> u32 {
59
+ // GQA: multiple query heads share one KV head
50
60
  let heads_per_kv = u.num_heads / u.num_kv_heads;
51
61
  return query_head_idx / heads_per_kv;
52
62
  }
53
63
 
64
+ fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
65
+ if (u.bidirectional_span_length == 0u) {
66
+ return false;
67
+ }
68
+ let span_start = u.bidirectional_span_start;
69
+ let span_end = span_start + u.bidirectional_span_length;
70
+ return abs_query >= span_start
71
+ && abs_query < span_end
72
+ && abs_key >= span_start
73
+ && abs_key < span_end;
74
+ }
75
+
76
+ // Check if position should be masked (causal + sliding window attention)
54
77
  fn is_masked(query_pos: u32, key_pos: u32) -> bool {
78
+ // Compute absolute positions
55
79
  let abs_query = query_pos + u.start_pos;
56
80
  let abs_key = u.kv_start + key_pos;
57
- if (u.is_causal != 0u && abs_key > abs_query) { return true; }
81
+
82
+ // Causal mask: query can only attend to keys at same or earlier positions
83
+ if (u.is_causal != 0u && abs_key > abs_query) {
84
+ if (is_bidirectional_span_visible(abs_query, abs_key)) {
85
+ return false;
86
+ }
87
+ return true;
88
+ }
89
+
90
+ // Sliding window mask: query can only attend to keys within the window
91
+ // Key must be >= (query - window + 1), i.e., within the last `window` positions
58
92
  if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
59
- if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
93
+ if (abs_key < abs_query - u.sliding_window + 1u) {
94
+ return true; // Key is too far in the past
95
+ }
60
96
  }
97
+
61
98
  return false;
62
99
  }
63
100
 
@@ -82,9 +119,11 @@ fn get_kv_len() -> u32 {
82
119
  return kv_len_buffer[0];
83
120
  }
84
121
 
122
+ // Main attention kernel - one workgroup per (query_block, head)
123
+ // Workgroups are dispatched linearly as: numQueryBlocks * numHeads.
124
+ // head_idx and query_block_idx are derived from workgroup_id.x.
85
125
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
86
126
  fn main(
87
- @builtin(global_invocation_id) global_id: vec3<u32>,
88
127
  @builtin(local_invocation_id) local_id: vec3<u32>,
89
128
  @builtin(workgroup_id) wg_id: vec3<u32>
90
129
  ) {
@@ -102,17 +141,21 @@ fn main(
102
141
  let query_len = u.query_len;
103
142
  let scale = u.scale;
104
143
 
144
+ // Query position this thread handles
105
145
  let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
106
146
  let valid_query = query_pos < query_len;
107
147
 
108
- var m_i: f32 = -3.402823e+38;
109
- var l_i: f32 = 0.0;
110
- var acc: array<f32, MAX_HEAD_DIM>;
148
+ // Initialize online softmax accumulators
149
+ var m_i: f32 = -3.402823e+38; // -inf for max tracking
150
+ var l_i: f32 = 0.0; // Sum of exp(x - max)
151
+ var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
111
152
 
153
+ // Initialize accumulator
112
154
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
113
155
  acc[d] = 0.0;
114
156
  }
115
157
 
158
+ // Load query for this thread into registers
116
159
  var q_local: array<f32, MAX_HEAD_DIM>;
117
160
  if (valid_query) {
118
161
  let q_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
@@ -121,11 +164,13 @@ fn main(
121
164
  }
122
165
  }
123
166
 
167
+ // Process key-value blocks
124
168
  let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
125
169
 
126
170
  for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
127
171
  let kv_block_start = kv_block * BLOCK_SIZE;
128
172
 
173
+ // Collaborative load of K block into shared memory
129
174
  let k_load_idx = kv_block_start + thread_idx;
130
175
  if (k_load_idx < seq_len) {
131
176
  let k_idx = get_kv_pos(k_load_idx);
@@ -139,6 +184,7 @@ fn main(
139
184
  }
140
185
  }
141
186
 
187
+ // Load V block
142
188
  let v_load_idx = kv_block_start + thread_idx;
143
189
  if (v_load_idx < seq_len) {
144
190
  let v_idx = get_kv_pos(v_load_idx);
@@ -154,20 +200,26 @@ fn main(
154
200
 
155
201
  workgroupBarrier();
156
202
 
203
+ // Compute attention scores for this block
157
204
  if (valid_query) {
205
+ // Find max in this block (for numerical stability)
158
206
  var block_max: f32 = -3.402823e+38;
159
207
 
160
208
  for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
161
209
  let key_pos = kv_block_start + k;
162
210
  if (key_pos >= seq_len) { continue; }
211
+
212
+ // Check causal mask
163
213
  if (is_masked(query_pos, key_pos)) { continue; }
164
214
 
215
+ // Compute Q @ K^T for this position
165
216
  var score: f32 = 0.0;
166
217
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
167
218
  score = score + q_local[d] * shared_K[k * head_dim + d];
168
219
  }
169
220
  score = score * scale;
170
221
 
222
+ // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
171
223
  if (u.attn_softcap > 0.0) {
172
224
  score = tanh(score / u.attn_softcap) * u.attn_softcap;
173
225
  }
@@ -176,14 +228,17 @@ fn main(
176
228
  shared_scores[thread_idx * BLOCK_SIZE + k] = score;
177
229
  }
178
230
 
231
+ // Online softmax update
179
232
  let m_new = max(m_i, block_max);
180
233
  let correction = exp(m_i - m_new);
181
234
 
235
+ // Rescale previous accumulator
182
236
  l_i = l_i * correction;
183
237
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
184
238
  acc[d] = acc[d] * correction;
185
239
  }
186
240
 
241
+ // Add contribution from this block
187
242
  for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
188
243
  let key_pos = kv_block_start + k;
189
244
  if (key_pos >= seq_len) { continue; }
@@ -193,6 +248,7 @@ fn main(
193
248
  let p = exp(score - m_new);
194
249
  l_i = l_i + p;
195
250
 
251
+ // Accumulate V contribution
196
252
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
197
253
  acc[d] = acc[d] + p * shared_V[k * head_dim + d];
198
254
  }
@@ -204,6 +260,8 @@ fn main(
204
260
  workgroupBarrier();
205
261
  }
206
262
 
263
+ // Normalize by sum and write output
264
+ // Always write output when valid_query - write zeros if l_i is 0 to avoid garbage
207
265
  if (valid_query) {
208
266
  let out_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
209
267
  let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
@@ -1,20 +1,23 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
3
- // Fused Multi-Head Attention Kernel (f16 KV)
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
+ // Fused Multi-Head Attention Kernel
4
4
  //
5
- // Same as attention.wgsl but K/V are stored as f16 for KV-cache compression.
6
- // Q and output remain f32; K/V are cast to f32 on load.
5
+ // Implements fused Q @ K^T → scale → mask → softmax → @ V
6
+ // Uses tiled/blocked approach to avoid materializing full attention matrix.
7
+ // Supports grouped query attention (GQA) where numKVHeads < numHeads.
8
+ //
9
+ // Based on Flash Attention principles adapted for WebGPU.
7
10
 
8
11
  enable f16;
9
12
 
10
13
  // Tile sizes for blocked attention
11
- const MAX_BLOCK_SIZE: u32 = 64u;
14
+ const MAX_BLOCK_SIZE: u32 = 32u;
12
15
  const MAX_HEAD_TILE: u32 = 64u;
13
16
  const MAX_HEAD_DIM: u32 = 64u;
14
17
 
15
- override BLOCK_SIZE: u32 = 64u; // Sequence tile size (must match WORKGROUP_SIZE)
18
+ override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
16
19
  override HEAD_TILE: u32 = 64u; // Head dimension tile
17
- override WORKGROUP_SIZE: u32 = 64u; // One thread per query position
20
+ override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
18
21
 
19
22
  struct Uniforms {
20
23
  num_heads: u32, // Number of query heads
@@ -31,7 +34,11 @@ struct Uniforms {
31
34
  kv_start: u32,
32
35
  page_size: u32,
33
36
  kv_layout: u32,
34
- _pad: u32,
37
+ bidirectional_span_start: u32,
38
+ bidirectional_span_length: u32,
39
+ _pad0: u32,
40
+ _pad1: u32,
41
+ _pad2: u32,
35
42
  }
36
43
 
37
44
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -54,16 +61,40 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
54
61
  return query_head_idx / heads_per_kv;
55
62
  }
56
63
 
64
+ fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
65
+ if (u.bidirectional_span_length == 0u) {
66
+ return false;
67
+ }
68
+ let span_start = u.bidirectional_span_start;
69
+ let span_end = span_start + u.bidirectional_span_length;
70
+ return abs_query >= span_start
71
+ && abs_query < span_end
72
+ && abs_key >= span_start
73
+ && abs_key < span_end;
74
+ }
75
+
57
76
  // Check if position should be masked (causal + sliding window attention)
58
77
  fn is_masked(query_pos: u32, key_pos: u32) -> bool {
78
+ // Compute absolute positions
59
79
  let abs_query = query_pos + u.start_pos;
60
80
  let abs_key = u.kv_start + key_pos;
61
- // Causal mask
62
- if (u.is_causal != 0u && abs_key > abs_query) { return true; }
63
- // Sliding window mask
81
+
82
+ // Causal mask: query can only attend to keys at same or earlier positions
83
+ if (u.is_causal != 0u && abs_key > abs_query) {
84
+ if (is_bidirectional_span_visible(abs_query, abs_key)) {
85
+ return false;
86
+ }
87
+ return true;
88
+ }
89
+
90
+ // Sliding window mask: query can only attend to keys within the window
91
+ // Key must be >= (query - window + 1), i.e., within the last `window` positions
64
92
  if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
65
- if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
93
+ if (abs_key < abs_query - u.sliding_window + 1u) {
94
+ return true; // Key is too far in the past
95
+ }
66
96
  }
97
+
67
98
  return false;
68
99
  }
69
100
 
@@ -93,7 +124,6 @@ fn get_kv_len() -> u32 {
93
124
  // head_idx and query_block_idx are derived from workgroup_id.x.
94
125
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
95
126
  fn main(
96
- @builtin(global_invocation_id) global_id: vec3<u32>,
97
127
  @builtin(local_invocation_id) local_id: vec3<u32>,
98
128
  @builtin(workgroup_id) wg_id: vec3<u32>
99
129
  ) {
@@ -118,7 +148,7 @@ fn main(
118
148
  // Initialize online softmax accumulators
119
149
  var m_i: f32 = -3.402823e+38; // -inf for max tracking
120
150
  var l_i: f32 = 0.0; // Sum of exp(x - max)
121
- var acc: array<f32, MAX_HEAD_DIM>;
151
+ var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
122
152
 
123
153
  // Initialize accumulator
124
154
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -189,7 +219,7 @@ fn main(
189
219
  }
190
220
  score = score * scale;
191
221
 
192
- // Gemma 2 attention softcapping
222
+ // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
193
223
  if (u.attn_softcap > 0.0) {
194
224
  score = tanh(score / u.attn_softcap) * u.attn_softcap;
195
225
  }