doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_streaming.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Streaming Multi-Head Attention Kernel (f16 KV, no workgroup storage)
4
4
  //
5
5
  // Same as attention_streaming.wgsl but K/V are stored as f16.
@@ -24,7 +24,11 @@ struct Uniforms {
24
24
  kv_start: u32,
25
25
  page_size: u32,
26
26
  kv_layout: u32,
27
- _pad: u32,
27
+ bidirectional_span_start: u32,
28
+ bidirectional_span_length: u32,
29
+ _pad0: u32,
30
+ _pad1: u32,
31
+ _pad2: u32,
28
32
  }
29
33
 
30
34
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -40,11 +44,26 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
40
44
  return query_head_idx / heads_per_kv;
41
45
  }
42
46
 
47
+ fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
48
+ if (u.bidirectional_span_length == 0u) {
49
+ return false;
50
+ }
51
+ let span_start = u.bidirectional_span_start;
52
+ let span_end = span_start + u.bidirectional_span_length;
53
+ return abs_query >= span_start
54
+ && abs_query < span_end
55
+ && abs_key >= span_start
56
+ && abs_key < span_end;
57
+ }
58
+
43
59
  fn is_masked(query_pos: u32, key_pos: u32) -> bool {
44
60
  let abs_query = query_pos + u.start_pos;
45
61
  let abs_key = u.kv_start + key_pos;
46
62
  // Causal mask
47
- if (u.is_causal != 0u && abs_key > abs_query) { return true; }
63
+ if (u.is_causal != 0u && abs_key > abs_query) {
64
+ if (is_bidirectional_span_visible(abs_query, abs_key)) { return false; }
65
+ return true;
66
+ }
48
67
  // Sliding window mask
49
68
  if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
50
69
  if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
@@ -2,16 +2,22 @@ export { runEmbedBackward, recordEmbedBackward } from './embed_backward.js';
2
2
  export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
3
3
  export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
4
4
  export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
5
+ export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
5
6
  export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
7
+ export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
6
8
  export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
7
9
  export { runRoPEBackward, recordRoPEBackward } from './rope_backward.js';
10
+ export { runRoPEBackward as runRopeBackward, recordRoPEBackward as recordRopeBackward } from './rope_backward.js';
8
11
  export { runSiluBackward, recordSiluBackward } from './silu_backward.js';
9
12
  export { runGeluBackward, recordGeluBackward } from './gelu_backward.js';
10
13
  export { runScaleBackward, recordScaleBackward } from './scale_backward.js';
11
14
  export { runCrossEntropyBackward, recordCrossEntropyBackward } from './cross_entropy_backward.js';
12
15
  export { runBiasAddBackward, recordBiasAddBackward } from './bias_add_backward.js';
13
16
  export { runUpsample2DBackward, recordUpsample2DBackward } from './upsample2d_backward.js';
17
+ export { runUpsample2DBackward as runUpsample2dBackward, recordUpsample2DBackward as recordUpsample2dBackward } from './upsample2d_backward.js';
14
18
  export { runPixelShuffleBackward, recordPixelShuffleBackward } from './pixel_shuffle_backward.js';
15
19
  export { runGroupNormBackward, recordGroupNormBackward } from './groupnorm_backward.js';
20
+ export { runGroupNormBackward as runGroupnormBackward, recordGroupNormBackward as recordGroupnormBackward } from './groupnorm_backward.js';
16
21
  export { runConv2DBackward, recordConv2DBackward } from './conv2d_backward.js';
22
+ export { runConv2DBackward as runConv2dBackward, recordConv2DBackward as recordConv2dBackward } from './conv2d_backward.js';
17
23
  export { runAdam, recordAdam } from './adam.js';
@@ -27,7 +27,6 @@
27
27
 
28
28
  override WORKGROUP_SIZE: u32 = 256u;
29
29
  const MAX_WORKGROUP_SIZE: u32 = 256u;
30
- override PARAMS_IS_F16: bool = false; // Weight/bias packed as f16 pairs (unused for now as we read f32)
31
30
 
32
31
  // =============================================================================
33
32
  // Uniforms
@@ -43,3 +43,38 @@ export declare function recordMatmulTransposeA(
43
43
  K: number,
44
44
  options?: { alpha?: number; outputBuffer?: GPUBuffer | null }
45
45
  ): Promise<Tensor>;
46
+
47
+ /**
48
+ * Factory that builds a backward-kernel runner pair from an op name
49
+ * and a small spec (uniform-writer + workgroup sizing).
50
+ */
51
+ export declare function createBackwardKernel(
52
+ opName: string,
53
+ spec: Record<string, unknown>
54
+ ): {
55
+ run: (...args: unknown[]) => Promise<Tensor>;
56
+ record: (recorder: CommandRecorder, ...args: unknown[]) => Promise<Tensor>;
57
+ };
58
+
59
+ /**
60
+ * Matmul-backward gradient w.r.t. input (dX = dY · Wᵀ). Submit + wait.
61
+ */
62
+ export declare function runMatmulBackwardDx(
63
+ dY: Tensor,
64
+ W: Tensor,
65
+ M: number,
66
+ K: number,
67
+ N: number,
68
+ options?: { outputBuffer?: GPUBuffer | null }
69
+ ): Promise<Tensor>;
70
+
71
+ /** Record runMatmulBackwardDx onto a CommandRecorder. */
72
+ export declare function recordMatmulBackwardDx(
73
+ recorder: CommandRecorder,
74
+ dY: Tensor,
75
+ W: Tensor,
76
+ M: number,
77
+ K: number,
78
+ N: number,
79
+ options?: { outputBuffer?: GPUBuffer | null }
80
+ ): Promise<Tensor>;
@@ -52,19 +52,3 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
52
52
  }
53
53
  }
54
54
 
55
- // Single-element version for odd-sized tensors
56
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
57
- fn main_single(@builtin(global_invocation_id) global_id: vec3<u32>) {
58
- let idx = global_id.x;
59
-
60
- if (idx >= u.num_elements) {
61
- return;
62
- }
63
-
64
- // Read as u16 pairs, extract the right one
65
- let pair_idx = idx / 2u;
66
- let packed = input[pair_idx];
67
-
68
- let bf16 = select(packed & 0xFFFFu, (packed >> 16u) & 0xFFFFu, (idx & 1u) == 1u);
69
- output[idx] = bitcast<f32>(bf16 << 16u);
70
- }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/bias_add.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // bias_add_f16.wgsl
4
4
 
5
5
  /**
@@ -7,7 +7,6 @@
7
7
  enable f16;
8
8
 
9
9
  override WORKGROUP_SIZE: u32 = 256u;
10
- const MAX_WG_X: u32 = 65535u;
11
10
 
12
11
  struct Uniforms {
13
12
  size: u32,
@@ -1,59 +1,15 @@
1
1
  --- src/gpu/kernels/attention.wgsl
2
2
  +++ src/gpu/kernels/attention_f16.wgsl
3
- @@ -1,32 +1,31 @@
4
- -// Fused Multi-Head Attention Kernel
5
- +// Fused Multi-Head Attention Kernel (f16 QKV + f16 output)
3
+ @@ -6,6 +6,8 @@
6
4
  //
7
- -// Implements fused Q @ K^T → scale → mask → softmax → @ V
8
- -// Uses tiled/blocked approach to avoid materializing full attention matrix.
9
- -// Supports grouped query attention (GQA) where numKVHeads < numHeads.
10
- -//
11
- -// Based on Flash Attention principles adapted for WebGPU.
12
- +// Same algorithm as attention.wgsl but Q/K/V are f16 and output is f16.
13
- +// Computation is performed in f32 for stability.
14
- +
15
- +enable f16;
5
+ // Based on Flash Attention principles adapted for WebGPU.
16
6
 
7
+ +enable f16;
8
+ +
17
9
  // Tile sizes for blocked attention
18
- -const MAX_BLOCK_SIZE: u32 = 32u;
19
- +const MAX_BLOCK_SIZE: u32 = 64u;
10
+ const MAX_BLOCK_SIZE: u32 = 32u;
20
11
  const MAX_HEAD_TILE: u32 = 64u;
21
- const MAX_HEAD_DIM: u32 = 64u;
22
-
23
- -override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
24
- -override HEAD_TILE: u32 = 64u; // Head dimension tile
25
- -override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
26
- +override BLOCK_SIZE: u32 = 64u;
27
- +override HEAD_TILE: u32 = 64u;
28
- +override WORKGROUP_SIZE: u32 = 64u;
29
-
30
- struct Uniforms {
31
- - num_heads: u32, // Number of query heads
32
- - num_kv_heads: u32, // Number of KV heads (for GQA)
33
- - head_dim: u32, // Dimension per head
34
- - seq_len: u32, // Current sequence length (for KV)
35
- - query_len: u32, // Query length (1 for decode, seq_len for prefill)
36
- - scale: f32, // 1/sqrt(head_dim)
37
- - is_causal: u32, // Apply causal mask (1 = yes)
38
- - start_pos: u32, // Absolute position offset for causal masking
39
- - attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
40
- - sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
41
- - kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
42
- + num_heads: u32,
43
- + num_kv_heads: u32,
44
- + head_dim: u32,
45
- + seq_len: u32,
46
- + query_len: u32,
47
- + scale: f32,
48
- + is_causal: u32,
49
- + start_pos: u32,
50
- + attn_softcap: f32,
51
- + sliding_window: u32,
52
- + kv_len_source: u32,
53
- kv_start: u32,
54
- page_size: u32,
55
- kv_layout: u32,
56
- @@ -34,44 +33,29 @@
12
+ @@ -38,10 +40,10 @@
57
13
  }
58
14
 
59
15
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -61,107 +17,23 @@
61
17
  -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
62
18
  -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
63
19
  -@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
64
- +@group(0) @binding(1) var<storage, read> Q: array<f16>;
65
- +@group(0) @binding(2) var<storage, read> K: array<f16>;
66
- +@group(0) @binding(3) var<storage, read> V: array<f16>;
67
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>;
20
+ +@group(0) @binding(1) var<storage, read> Q: array<f16>; // [query_len, num_heads, head_dim]
21
+ +@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
22
+ +@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
23
+ +@group(0) @binding(4) var<storage, read_write> output: array<f16>; // [query_len, num_heads, head_dim]
68
24
  @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
69
25
  @group(0) @binding(6) var<storage, read> page_table: array<u32>;
70
26
 
71
- -// Shared memory for tiled computation
72
- var<workgroup> shared_K: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
73
- var<workgroup> shared_V: array<f32, MAX_BLOCK_SIZE * MAX_HEAD_TILE>;
74
- var<workgroup> shared_scores: array<f32, MAX_BLOCK_SIZE * MAX_BLOCK_SIZE>;
75
-
76
- -// Get KV head index for grouped query attention
77
- fn get_kv_head_idx(query_head_idx: u32) -> u32 {
78
- - // GQA: multiple query heads share one KV head
79
- let heads_per_kv = u.num_heads / u.num_kv_heads;
80
- return query_head_idx / heads_per_kv;
81
- }
82
-
83
- -// Check if position should be masked (causal + sliding window attention)
84
- fn is_masked(query_pos: u32, key_pos: u32) -> bool {
85
- - // Compute absolute positions
86
- let abs_query = query_pos + u.start_pos;
87
- let abs_key = u.kv_start + key_pos;
88
- -
89
- - // Causal mask: query can only attend to keys at same or earlier positions
90
- - if (u.is_causal != 0u && abs_key > abs_query) {
91
- - return true;
92
- - }
93
- -
94
- - // Sliding window mask: query can only attend to keys within the window
95
- - // Key must be >= (query - window + 1), i.e., within the last `window` positions
96
- + if (u.is_causal != 0u && abs_key > abs_query) { return true; }
97
- if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
98
- - if (abs_key < abs_query - u.sliding_window + 1u) {
99
- - return true; // Key is too far in the past
100
- - }
101
- - }
102
- -
103
- + if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
104
- + }
105
- return false;
106
- }
107
-
108
- @@ -96,11 +80,9 @@
109
- return kv_len_buffer[0];
110
- }
111
-
112
- -// Main attention kernel - one workgroup per (query_block, head)
113
- -// Workgroups are dispatched linearly as: numQueryBlocks * numHeads.
114
- -// head_idx and query_block_idx are derived from workgroup_id.x.
115
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
116
- fn main(
117
- + @builtin(global_invocation_id) global_id: vec3<u32>,
118
- @builtin(local_invocation_id) local_id: vec3<u32>,
119
- @builtin(workgroup_id) wg_id: vec3<u32>
120
- ) {
121
- @@ -118,42 +100,36 @@
122
- let query_len = u.query_len;
123
- let scale = u.scale;
124
-
125
- - // Query position this thread handles
126
- let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
127
- let valid_query = query_pos < query_len;
128
-
129
- - // Initialize online softmax accumulators
130
- - var m_i: f32 = -3.402823e+38; // -inf for max tracking
131
- - var l_i: f32 = 0.0; // Sum of exp(x - max)
132
- - var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
133
- -
134
- - // Initialize accumulator
135
- + var m_i: f32 = -3.402823e+38;
136
- + var l_i: f32 = 0.0;
137
- + var acc: array<f32, MAX_HEAD_DIM>;
138
- +
139
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
140
- acc[d] = 0.0;
141
- }
142
-
143
- - // Load query for this thread into registers
144
- var q_local: array<f32, MAX_HEAD_DIM>;
27
+ @@ -156,7 +158,7 @@
145
28
  if (valid_query) {
146
29
  let q_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
147
30
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
148
31
  - q_local[d] = Q[q_offset + d];
149
- - }
150
- - }
151
- -
152
- - // Process key-value blocks
153
32
  + q_local[d] = f32(Q[q_offset + d]);
154
- + }
155
- + }
156
- +
157
- let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
158
-
159
- for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
160
- let kv_block_start = kv_block * BLOCK_SIZE;
33
+ }
34
+ }
161
35
 
162
- - // Collaborative load of K block into shared memory
163
- let k_load_idx = kv_block_start + thread_idx;
164
- if (k_load_idx < seq_len) {
36
+ @@ -172,7 +174,7 @@
165
37
  let k_idx = get_kv_pos(k_load_idx);
166
38
  let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
167
39
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -170,13 +42,7 @@
170
42
  }
171
43
  } else {
172
44
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
173
- @@ -161,13 +137,12 @@
174
- }
175
- }
176
-
177
- - // Load V block
178
- let v_load_idx = kv_block_start + thread_idx;
179
- if (v_load_idx < seq_len) {
45
+ @@ -186,7 +188,7 @@
180
46
  let v_idx = get_kv_pos(v_load_idx);
181
47
  let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
182
48
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -185,74 +51,12 @@
185
51
  }
186
52
  } else {
187
53
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
188
- @@ -177,26 +152,20 @@
189
-
190
- workgroupBarrier();
191
-
192
- - // Compute attention scores for this block
193
- if (valid_query) {
194
- - // Find max in this block (for numerical stability)
195
- var block_max: f32 = -3.402823e+38;
196
-
197
- for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
198
- let key_pos = kv_block_start + k;
199
- if (key_pos >= seq_len) { continue; }
200
- -
201
- - // Check causal mask
202
- if (is_masked(query_pos, key_pos)) { continue; }
203
-
204
- - // Compute Q @ K^T for this position
205
- var score: f32 = 0.0;
206
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
207
- score = score + q_local[d] * shared_K[k * head_dim + d];
208
- }
209
- score = score * scale;
210
-
211
- - // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
212
- if (u.attn_softcap > 0.0) {
213
- score = tanh(score / u.attn_softcap) * u.attn_softcap;
214
- }
215
- @@ -205,17 +174,14 @@
216
- shared_scores[thread_idx * BLOCK_SIZE + k] = score;
217
- }
218
-
219
- - // Online softmax update
220
- let m_new = max(m_i, block_max);
221
- let correction = exp(m_i - m_new);
222
-
223
- - // Rescale previous accumulator
224
- l_i = l_i * correction;
225
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
226
- acc[d] = acc[d] * correction;
227
- }
228
-
229
- - // Add contribution from this block
230
- for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
231
- let key_pos = kv_block_start + k;
232
- if (key_pos >= seq_len) { continue; }
233
- @@ -225,7 +191,6 @@
234
- let p = exp(score - m_new);
235
- l_i = l_i + p;
236
-
237
- - // Accumulate V contribution
238
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
239
- acc[d] = acc[d] + p * shared_V[k * head_dim + d];
240
- }
241
- @@ -237,13 +202,11 @@
242
- workgroupBarrier();
243
- }
244
-
245
- - // Normalize by sum and write output
246
- - // Always write output when valid_query - write zeros if l_i is 0 to avoid garbage
247
- if (valid_query) {
54
+ @@ -262,7 +264,7 @@
248
55
  let out_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
249
56
  let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
250
57
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
251
58
  - output[out_offset + d] = acc[d] * inv_l_i;
252
- - }
253
- - }
254
- -}
255
59
  + output[out_offset + d] = f16(acc[d] * inv_l_i);
256
- + }
257
- + }
258
- +}
60
+ }
61
+ }
62
+ }
@@ -1,34 +1,15 @@
1
1
  --- src/gpu/kernels/attention.wgsl
2
2
  +++ src/gpu/kernels/attention_f16kv.wgsl
3
- @@ -1,19 +1,18 @@
4
- -// Fused Multi-Head Attention Kernel
5
- +// Fused Multi-Head Attention Kernel (f16 KV)
3
+ @@ -6,6 +6,8 @@
6
4
  //
7
- -// Implements fused Q @ K^T → scale → mask → softmax → @ V
8
- -// Uses tiled/blocked approach to avoid materializing full attention matrix.
9
- -// Supports grouped query attention (GQA) where numKVHeads < numHeads.
10
- -//
11
- -// Based on Flash Attention principles adapted for WebGPU.
12
- +// Same as attention.wgsl but K/V are stored as f16 for KV-cache compression.
13
- +// Q and output remain f32; K/V are cast to f32 on load.
14
- +
15
- +enable f16;
5
+ // Based on Flash Attention principles adapted for WebGPU.
16
6
 
7
+ +enable f16;
8
+ +
17
9
  // Tile sizes for blocked attention
18
- -const MAX_BLOCK_SIZE: u32 = 32u;
19
- +const MAX_BLOCK_SIZE: u32 = 64u;
10
+ const MAX_BLOCK_SIZE: u32 = 32u;
20
11
  const MAX_HEAD_TILE: u32 = 64u;
21
- const MAX_HEAD_DIM: u32 = 64u;
22
-
23
- -override BLOCK_SIZE: u32 = 32u; // Sequence tile size (must match WORKGROUP_SIZE)
24
- +override BLOCK_SIZE: u32 = 64u; // Sequence tile size (must match WORKGROUP_SIZE)
25
- override HEAD_TILE: u32 = 64u; // Head dimension tile
26
- -override WORKGROUP_SIZE: u32 = 32u; // One thread per query position
27
- +override WORKGROUP_SIZE: u32 = 64u; // One thread per query position
28
-
29
- struct Uniforms {
30
- num_heads: u32, // Number of query heads
31
- @@ -35,8 +34,8 @@
12
+ @@ -39,8 +41,8 @@
32
13
 
33
14
  @group(0) @binding(0) var<uniform> u: Uniforms;
34
15
  @group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
@@ -39,53 +20,7 @@
39
20
  @group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
40
21
  @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
41
22
  @group(0) @binding(6) var<storage, read> page_table: array<u32>;
42
- @@ -55,23 +54,14 @@
43
-
44
- // Check if position should be masked (causal + sliding window attention)
45
- fn is_masked(query_pos: u32, key_pos: u32) -> bool {
46
- - // Compute absolute positions
47
- let abs_query = query_pos + u.start_pos;
48
- let abs_key = u.kv_start + key_pos;
49
- -
50
- - // Causal mask: query can only attend to keys at same or earlier positions
51
- - if (u.is_causal != 0u && abs_key > abs_query) {
52
- - return true;
53
- - }
54
- -
55
- - // Sliding window mask: query can only attend to keys within the window
56
- - // Key must be >= (query - window + 1), i.e., within the last `window` positions
57
- + // Causal mask
58
- + if (u.is_causal != 0u && abs_key > abs_query) { return true; }
59
- + // Sliding window mask
60
- if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
61
- - if (abs_key < abs_query - u.sliding_window + 1u) {
62
- - return true; // Key is too far in the past
63
- - }
64
- - }
65
- -
66
- + if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
67
- + }
68
- return false;
69
- }
70
-
71
- @@ -101,6 +91,7 @@
72
- // head_idx and query_block_idx are derived from workgroup_id.x.
73
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
74
- fn main(
75
- + @builtin(global_invocation_id) global_id: vec3<u32>,
76
- @builtin(local_invocation_id) local_id: vec3<u32>,
77
- @builtin(workgroup_id) wg_id: vec3<u32>
78
- ) {
79
- @@ -125,7 +116,7 @@
80
- // Initialize online softmax accumulators
81
- var m_i: f32 = -3.402823e+38; // -inf for max tracking
82
- var l_i: f32 = 0.0; // Sum of exp(x - max)
83
- - var acc: array<f32, MAX_HEAD_DIM>; // Accumulator for output [head_dim], assuming head_dim <= MAX_HEAD_DIM
84
- + var acc: array<f32, MAX_HEAD_DIM>;
85
-
86
- // Initialize accumulator
87
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
88
- @@ -153,7 +144,7 @@
23
+ @@ -172,7 +174,7 @@
89
24
  let k_idx = get_kv_pos(k_load_idx);
90
25
  let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
91
26
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -94,7 +29,7 @@
94
29
  }
95
30
  } else {
96
31
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
97
- @@ -167,7 +158,7 @@
32
+ @@ -186,7 +188,7 @@
98
33
  let v_idx = get_kv_pos(v_load_idx);
99
34
  let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
100
35
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -103,12 +38,3 @@
103
38
  }
104
39
  } else {
105
40
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
106
- @@ -196,7 +187,7 @@
107
- }
108
- score = score * scale;
109
-
110
- - // Gemma 2 attention softcapping: score = tanh(score / softcap) * softcap
111
- + // Gemma 2 attention softcapping
112
- if (u.attn_softcap > 0.0) {
113
- score = tanh(score / u.attn_softcap) * u.attn_softcap;
114
- }