doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,306 @@
1
+ // Flash-Attention-Style Prefill Kernel (head_dim=256, f16 KV)
2
+ //
3
+ // Flash attention with KV-axis splitting to raise RDNA3 occupancy. The existing
4
+ // attention_head256_f16kv.wgsl dispatches (num_query_blocks × num_heads, 1, 1)
5
+ // workgroups, which is too few for RDNA3 at common prefill shapes (e.g. 16
6
+ // workgroups for prefill=64, 8 heads). This variant multiplies the workgroup
7
+ // count by num_kv_splits along the KV axis and produces partial (output, m, l)
8
+ // per split — a companion reduction kernel merges them.
9
+ //
10
+ // Algorithm: online-softmax flash attention per split.
11
+ //
12
+ // Pass 1 (this kernel): each workgroup processes one KV slice for its
13
+ // (query_block, head) pair; writes un-normalised acc + m + l to partial buffers.
14
+ // Pass 2 (flash_reduce): merges partials across splits for each (query, head).
15
+
16
+ enable f16;
17
+
18
+ const BLOCK_SIZE: u32 = 32u;
19
+ const WORKGROUP_SIZE: u32 = BLOCK_SIZE;
20
+ const HEAD_DIM: u32 = 256u;
21
+ const HEAD_DIM_VECS: u32 = 64u;
22
+
23
+ struct Uniforms {
24
+ num_heads: u32,
25
+ num_kv_heads: u32,
26
+ head_dim: u32,
27
+ seq_len: u32,
28
+ query_len: u32,
29
+ scale: f32,
30
+ is_causal: u32,
31
+ start_pos: u32,
32
+ attn_softcap: f32,
33
+ sliding_window: u32,
34
+ kv_len_source: u32,
35
+ kv_start: u32,
36
+ page_size: u32,
37
+ kv_layout: u32,
38
+ num_kv_splits: u32,
39
+ _pad: u32,
40
+ }
41
+
42
+ @group(0) @binding(0) var<uniform> u: Uniforms;
43
+ @group(0) @binding(1) var<storage, read> Q: array<f32>;
44
+ @group(0) @binding(2) var<storage, read> K: array<f16>;
45
+ @group(0) @binding(3) var<storage, read> V: array<f16>;
46
+ // Per-split partials: [num_query_blocks, num_heads, num_kv_splits, BLOCK_SIZE, HEAD_DIM]
47
+ // When num_kv_splits == 1 (single-split fast path), this buffer is bound to the
48
+ // FINAL output buffer and we write normalised output directly — skipping the
49
+ // reduce pass entirely.
50
+ @group(0) @binding(4) var<storage, read_write> partial_acc: array<f32>;
51
+ // Per-split stats: [num_query_blocks, num_heads, num_kv_splits, BLOCK_SIZE]
52
+ @group(0) @binding(5) var<storage, read_write> partial_m: array<f32>;
53
+ @group(0) @binding(6) var<storage, read_write> partial_l: array<f32>;
54
+ @group(0) @binding(7) var<storage, read> kv_len_buffer: array<u32>;
55
+ @group(0) @binding(8) var<storage, read> page_table: array<u32>;
56
+
57
+ // Separate shared tiles for K and V so they can be loaded in parallel within
58
+ // the same loop iteration. Saves a workgroupBarrier per KV block vs the
59
+ // shared-tile-reuse pattern of the non-flash head256 kernel.
60
+ var<workgroup> shared_K: array<vec4<f16>, BLOCK_SIZE * HEAD_DIM_VECS>;
61
+ var<workgroup> shared_V: array<vec4<f16>, BLOCK_SIZE * HEAD_DIM_VECS>;
62
+
63
+ fn zero_vec4_f16() -> vec4<f16> {
64
+ return vec4<f16>(f16(0.0), f16(0.0), f16(0.0), f16(0.0));
65
+ }
66
+
67
+ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
68
+ let heads_per_kv = u.num_heads / u.num_kv_heads;
69
+ return query_head_idx / heads_per_kv;
70
+ }
71
+
72
+ fn get_kv_pos(key_pos: u32) -> u32 {
73
+ let abs_key = u.kv_start + key_pos;
74
+ if (u.kv_layout == 1u && u.sliding_window > 0u) {
75
+ return abs_key % u.sliding_window;
76
+ }
77
+ if (u.kv_layout == 2u) {
78
+ let page_idx = abs_key / u.page_size;
79
+ let in_page = abs_key - (page_idx * u.page_size);
80
+ let phys_page = page_table[page_idx];
81
+ return phys_page * u.page_size + in_page;
82
+ }
83
+ return abs_key;
84
+ }
85
+
86
+ fn get_kv_len() -> u32 {
87
+ if (u.kv_len_source == 0u) {
88
+ return u.seq_len;
89
+ }
90
+ return kv_len_buffer[0];
91
+ }
92
+
93
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
94
+ fn main(
95
+ @builtin(local_invocation_id) local_id: vec3<u32>,
96
+ @builtin(workgroup_id) wg_id: vec3<u32>
97
+ ) {
98
+ if (u.head_dim != HEAD_DIM) {
99
+ return;
100
+ }
101
+
102
+ // Dispatch layout: linear = (query_block_idx * num_heads + head_idx) * num_kv_splits + kv_split_idx
103
+ let num_heads = u.num_heads;
104
+ let num_kv_splits = u.num_kv_splits;
105
+ let linear = wg_id.x;
106
+ let kv_split_idx = linear % num_kv_splits;
107
+ let linear_qh = linear / num_kv_splits;
108
+ let head_idx = linear_qh % num_heads;
109
+ let query_block_idx = linear_qh / num_heads;
110
+ let thread_idx = local_id.x;
111
+
112
+ let kv_head_idx = get_kv_head_idx(head_idx);
113
+ let seq_len = get_kv_len();
114
+ let query_len = u.query_len;
115
+ let scale = u.scale;
116
+
117
+ let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
118
+ let valid_query = query_pos < query_len;
119
+ let abs_query = query_pos + u.start_pos;
120
+
121
+ // KV slice bounds for this split
122
+ let kv_per_split = (seq_len + num_kv_splits - 1u) / num_kv_splits;
123
+ let kv_slice_start = kv_split_idx * kv_per_split;
124
+ let kv_slice_end = min(kv_slice_start + kv_per_split, seq_len);
125
+
126
+ var q_local: array<vec4<f32>, HEAD_DIM_VECS>;
127
+ var acc: array<vec4<f32>, HEAD_DIM_VECS>;
128
+
129
+ if (valid_query) {
130
+ let q_offset = query_pos * num_heads * HEAD_DIM + head_idx * HEAD_DIM;
131
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
132
+ let base = q_offset + d4 * 4u;
133
+ q_local[d4] = vec4<f32>(Q[base], Q[base + 1u], Q[base + 2u], Q[base + 3u]);
134
+ acc[d4] = vec4<f32>(0.0);
135
+ }
136
+ }
137
+
138
+ var m_i: f32 = -3.402823e+38;
139
+ var l_i: f32 = 0.0;
140
+
141
+ // Clamp KV slice to causal/sliding constraints for this query
142
+ var min_key_pos: u32 = kv_slice_start;
143
+ var max_key_pos: u32 = kv_slice_end;
144
+
145
+ if (valid_query) {
146
+ if (u.is_causal != 0u) {
147
+ if (abs_query < u.kv_start) {
148
+ max_key_pos = min(max_key_pos, 0u);
149
+ } else {
150
+ let causal_limit = abs_query - u.kv_start + 1u;
151
+ max_key_pos = min(max_key_pos, causal_limit);
152
+ }
153
+ }
154
+ if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
155
+ let min_abs_key = abs_query - u.sliding_window + 1u;
156
+ if (min_abs_key > u.kv_start) {
157
+ let sw_min = min_abs_key - u.kv_start;
158
+ if (sw_min > min_key_pos) {
159
+ min_key_pos = sw_min;
160
+ }
161
+ }
162
+ min_key_pos = min(min_key_pos, seq_len);
163
+ }
164
+ }
165
+
166
+ // Iterate KV blocks WITHIN our split range
167
+ let num_kv_blocks_in_slice = (kv_slice_end - kv_slice_start + BLOCK_SIZE - 1u) / BLOCK_SIZE;
168
+
169
+ for (var kv_block: u32 = 0u; kv_block < num_kv_blocks_in_slice; kv_block = kv_block + 1u) {
170
+ let kv_block_start = kv_slice_start + kv_block * BLOCK_SIZE;
171
+
172
+ var scores: array<f32, BLOCK_SIZE>;
173
+ var key_active: array<u32, BLOCK_SIZE>;
174
+ var probs: array<f32, BLOCK_SIZE>;
175
+ for (var k_init: u32 = 0u; k_init < BLOCK_SIZE; k_init = k_init + 1u) {
176
+ scores[k_init] = 0.0;
177
+ key_active[k_init] = 0u;
178
+ probs[k_init] = 0.0;
179
+ }
180
+
181
+ if (valid_query) {
182
+ for (var k_mask: u32 = 0u; k_mask < BLOCK_SIZE; k_mask = k_mask + 1u) {
183
+ let key_pos = kv_block_start + k_mask;
184
+ if (key_pos < kv_slice_end && key_pos >= min_key_pos && key_pos < max_key_pos) {
185
+ key_active[k_mask] = 1u;
186
+ }
187
+ }
188
+ }
189
+
190
+ // Load K and V in parallel into separate shared tiles. A single barrier
191
+ // synchronises both loads before we score K and accumulate V.
192
+ let key_pos_load = kv_block_start + thread_idx;
193
+ let shared_row = thread_idx * HEAD_DIM_VECS;
194
+ if (key_pos_load < kv_slice_end) {
195
+ let k_idx = get_kv_pos(key_pos_load);
196
+ let k_offset = k_idx * u.num_kv_heads * HEAD_DIM + kv_head_idx * HEAD_DIM;
197
+ let v_idx = k_idx;
198
+ let v_offset = v_idx * u.num_kv_heads * HEAD_DIM + kv_head_idx * HEAD_DIM;
199
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
200
+ let kb = k_offset + d4 * 4u;
201
+ let vb = v_offset + d4 * 4u;
202
+ shared_K[shared_row + d4] = vec4<f16>(
203
+ K[kb], K[kb + 1u], K[kb + 2u], K[kb + 3u]
204
+ );
205
+ shared_V[shared_row + d4] = vec4<f16>(
206
+ V[vb], V[vb + 1u], V[vb + 2u], V[vb + 3u]
207
+ );
208
+ }
209
+ } else {
210
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
211
+ shared_K[shared_row + d4] = zero_vec4_f16();
212
+ shared_V[shared_row + d4] = zero_vec4_f16();
213
+ }
214
+ }
215
+
216
+ workgroupBarrier();
217
+
218
+ var m_new: f32 = m_i;
219
+ if (valid_query) {
220
+ var block_max: f32 = -3.402823e+38;
221
+ for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
222
+ if (key_active[k] == 0u) { continue; }
223
+ let key_row = k * HEAD_DIM_VECS;
224
+ var dot_partial: f32 = 0.0;
225
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
226
+ dot_partial = dot_partial + dot(
227
+ q_local[d4], vec4<f32>(shared_K[key_row + d4])
228
+ );
229
+ }
230
+ var s = dot_partial * scale;
231
+ if (u.attn_softcap > 0.0) {
232
+ s = tanh(s / u.attn_softcap) * u.attn_softcap;
233
+ }
234
+ scores[k] = s;
235
+ block_max = max(block_max, s);
236
+ }
237
+
238
+ m_new = max(m_i, block_max);
239
+ let correction = exp(m_i - m_new);
240
+ l_i = l_i * correction;
241
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
242
+ acc[d4] = acc[d4] * correction;
243
+ }
244
+
245
+ for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
246
+ if (key_active[k] == 0u) { continue; }
247
+ let p = exp(scores[k] - m_new);
248
+ probs[k] = p;
249
+ l_i = l_i + p;
250
+ }
251
+
252
+ // V is already in shared_V — apply immediately (no reload, no barrier).
253
+ for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
254
+ let p = probs[k];
255
+ if (p == 0.0) { continue; }
256
+ let value_row = k * HEAD_DIM_VECS;
257
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
258
+ acc[d4] = acc[d4] + p * vec4<f32>(shared_V[value_row + d4]);
259
+ }
260
+ }
261
+ m_i = m_new;
262
+ }
263
+
264
+ workgroupBarrier();
265
+ }
266
+
267
+ if (valid_query) {
268
+ let qh_idx = query_block_idx * num_heads + head_idx;
269
+ if (num_kv_splits == 1u) {
270
+ // Single-split fast path: skip the reduce pass entirely. Write the
271
+ // FINAL normalised output directly to the output buffer (which is
272
+ // bound in slot 4 under the partial_acc name). Layout matches
273
+ // [query_len, num_heads, head_dim].
274
+ let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
275
+ let out_offset = query_pos * num_heads * HEAD_DIM + head_idx * HEAD_DIM;
276
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
277
+ let base = out_offset + d4 * 4u;
278
+ let normed = acc[d4] * inv_l_i;
279
+ partial_acc[base] = normed.x;
280
+ partial_acc[base + 1u] = normed.y;
281
+ partial_acc[base + 2u] = normed.z;
282
+ partial_acc[base + 3u] = normed.w;
283
+ }
284
+ // m/l buffers are still bound but unused on the single-split path.
285
+ // Leave them untouched to avoid spurious writes.
286
+ } else {
287
+ // Multi-split: write UN-NORMALISED partial acc and per-query m/l for
288
+ // this split. Layout:
289
+ // partial_acc[(qb*num_heads + h) * num_kv_splits * BLOCK_SIZE * HEAD_DIM
290
+ // + kv_split * BLOCK_SIZE * HEAD_DIM
291
+ // + thread_idx * HEAD_DIM + d]
292
+ let partial_base = (qh_idx * num_kv_splits + kv_split_idx) * BLOCK_SIZE * HEAD_DIM
293
+ + thread_idx * HEAD_DIM;
294
+ for (var d4: u32 = 0u; d4 < HEAD_DIM_VECS; d4 = d4 + 1u) {
295
+ let base = partial_base + d4 * 4u;
296
+ partial_acc[base] = acc[d4].x;
297
+ partial_acc[base + 1u] = acc[d4].y;
298
+ partial_acc[base + 2u] = acc[d4].z;
299
+ partial_acc[base + 3u] = acc[d4].w;
300
+ }
301
+ let stats_idx = (qh_idx * num_kv_splits + kv_split_idx) * BLOCK_SIZE + thread_idx;
302
+ partial_m[stats_idx] = m_i;
303
+ partial_l[stats_idx] = l_i;
304
+ }
305
+ }
306
+ }
@@ -0,0 +1,101 @@
1
+ // Flash-Attention Reduction Kernel
2
+ //
3
+ // Merges per-split (partial_acc, m, l) produced by
4
+ // attention_prefill_flash_head256_f16kv.wgsl into the final attention output.
5
+ //
6
+ // Online softmax merge across splits: for each (query, head),
7
+ // m_global = max(m_s)
8
+ // l_global = sum(exp(m_s - m_global) * l_s)
9
+ // output[d] = sum(exp(m_s - m_global) * acc_s[d]) / l_global
10
+ //
11
+ // Dispatch: (ceil(query_len * num_heads / WORKGROUP_SIZE), HEAD_DIM_VECS, 1)
12
+ // Each thread owns one (query_head_flat, d4) output lane.
13
+
14
+ const HEAD_DIM: u32 = 256u;
15
+ const HEAD_DIM_VECS: u32 = 64u;
16
+ const BLOCK_SIZE: u32 = 32u;
17
+ const WORKGROUP_SIZE: u32 = 64u;
18
+ const MAX_KV_SPLITS: u32 = 16u;
19
+
20
+ struct ReduceUniforms {
21
+ num_heads: u32,
22
+ query_len: u32,
23
+ num_kv_splits: u32,
24
+ _pad: u32,
25
+ }
26
+
27
+ @group(0) @binding(0) var<uniform> u: ReduceUniforms;
28
+ @group(0) @binding(1) var<storage, read> partial_acc: array<f32>;
29
+ @group(0) @binding(2) var<storage, read> partial_m: array<f32>;
30
+ @group(0) @binding(3) var<storage, read> partial_l: array<f32>;
31
+ @group(0) @binding(4) var<storage, read_write> output: array<f32>;
32
+
33
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
34
+ fn main(
35
+ @builtin(global_invocation_id) global_id: vec3<u32>
36
+ ) {
37
+ let qh_flat = global_id.x;
38
+ let d4 = global_id.y;
39
+
40
+ let num_heads = u.num_heads;
41
+ let query_len = u.query_len;
42
+ let num_kv_splits = u.num_kv_splits;
43
+
44
+ let total_qh = query_len * num_heads;
45
+ if (qh_flat >= total_qh || d4 >= HEAD_DIM_VECS) {
46
+ return;
47
+ }
48
+
49
+ let query_pos = qh_flat / num_heads;
50
+ let head_idx = qh_flat % num_heads;
51
+ let query_block_idx = query_pos / BLOCK_SIZE;
52
+ let thread_within_block = query_pos - query_block_idx * BLOCK_SIZE;
53
+ let qh_idx = query_block_idx * num_heads + head_idx;
54
+
55
+ // Gather m_s and l_s across splits to find global m and global l.
56
+ var m_global: f32 = -3.402823e+38;
57
+ // Safe cap on splits so compilers accept the bounded loop.
58
+ let splits = min(num_kv_splits, MAX_KV_SPLITS);
59
+ for (var s: u32 = 0u; s < splits; s = s + 1u) {
60
+ let stats_idx = (qh_idx * num_kv_splits + s) * BLOCK_SIZE + thread_within_block;
61
+ let m_s = partial_m[stats_idx];
62
+ if (m_s > m_global) {
63
+ m_global = m_s;
64
+ }
65
+ }
66
+
67
+ // Compute global l as weighted sum of per-split l with exp correction.
68
+ var l_global: f32 = 0.0;
69
+ for (var s: u32 = 0u; s < splits; s = s + 1u) {
70
+ let stats_idx = (qh_idx * num_kv_splits + s) * BLOCK_SIZE + thread_within_block;
71
+ let m_s = partial_m[stats_idx];
72
+ let l_s = partial_l[stats_idx];
73
+ l_global = l_global + l_s * exp(m_s - m_global);
74
+ }
75
+
76
+ // Merge acc across splits for this d4.
77
+ var merged = vec4<f32>(0.0);
78
+ for (var s: u32 = 0u; s < splits; s = s + 1u) {
79
+ let stats_idx = (qh_idx * num_kv_splits + s) * BLOCK_SIZE + thread_within_block;
80
+ let m_s = partial_m[stats_idx];
81
+ let correction = exp(m_s - m_global);
82
+ let partial_base = (qh_idx * num_kv_splits + s) * BLOCK_SIZE * HEAD_DIM
83
+ + thread_within_block * HEAD_DIM
84
+ + d4 * 4u;
85
+ let partial_vec = vec4<f32>(
86
+ partial_acc[partial_base],
87
+ partial_acc[partial_base + 1u],
88
+ partial_acc[partial_base + 2u],
89
+ partial_acc[partial_base + 3u]
90
+ );
91
+ merged = merged + correction * partial_vec;
92
+ }
93
+
94
+ let inv_l = select(0.0, 1.0 / l_global, l_global > 0.0);
95
+ let out_offset = query_pos * num_heads * HEAD_DIM + head_idx * HEAD_DIM + d4 * 4u;
96
+ let final_vec = merged * inv_l;
97
+ output[out_offset] = final_vec.x;
98
+ output[out_offset + 1u] = final_vec.y;
99
+ output[out_offset + 2u] = final_vec.z;
100
+ output[out_offset + 3u] = final_vec.w;
101
+ }
@@ -1,9 +1,9 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_small.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused Multi-Head Attention Kernel (small tiles, f16 QKV + f16 output)
4
4
  //
5
5
  // Same algorithm as attention_small.wgsl but Q/K/V are f16 and output is f16.
6
- // Shared tiles use f16 to reduce workgroup storage; math uses f32.
6
+ // Shared tiles and attention arithmetic use f16 for the all-f16 lane.
7
7
 
8
8
  enable f16;
9
9
 
@@ -11,6 +11,7 @@ const BLOCK_SIZE: u32 = 32u;
11
11
  const WORKGROUP_SIZE: u32 = BLOCK_SIZE;
12
12
  const HEAD_TILE: u32 = 32u;
13
13
  const MAX_HEAD_DIM: u32 = 256u;
14
+ const NEG_INF = f16(-65504.0);
14
15
 
15
16
  struct Uniforms {
16
17
  num_heads: u32,
@@ -95,24 +96,25 @@ fn main(
95
96
  }
96
97
  let seq_len = get_kv_len();
97
98
  let query_len = u.query_len;
98
- let scale = u.scale;
99
+ let scale = f16(u.scale);
100
+ let softcap = f16(u.attn_softcap);
99
101
 
100
102
  let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
101
103
  let valid_query = query_pos < query_len;
102
104
 
103
- var q_local: array<f32, MAX_HEAD_DIM>;
104
- var acc: array<f32, MAX_HEAD_DIM>;
105
+ var q_local: array<f16, MAX_HEAD_DIM>;
106
+ var acc: array<f16, MAX_HEAD_DIM>;
105
107
 
106
108
  if (valid_query) {
107
109
  let q_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
108
110
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
109
- q_local[d] = f32(Q[q_offset + d]);
110
- acc[d] = 0.0;
111
+ q_local[d] = Q[q_offset + d];
112
+ acc[d] = f16(0.0);
111
113
  }
112
114
  }
113
115
 
114
- var m_i: f32 = -3.402823e+38;
115
- var l_i: f32 = 0.0;
116
+ var m_i: f16 = NEG_INF;
117
+ var l_i: f16 = f16(0.0);
116
118
 
117
119
  let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
118
120
  let num_head_tiles = (head_dim + HEAD_TILE - 1u) / HEAD_TILE;
@@ -120,9 +122,9 @@ fn main(
120
122
  for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
121
123
  let kv_block_start = kv_block * BLOCK_SIZE;
122
124
 
123
- var scores: array<f32, BLOCK_SIZE>;
125
+ var scores: array<f16, BLOCK_SIZE>;
124
126
  for (var k_init: u32 = 0u; k_init < BLOCK_SIZE; k_init = k_init + 1u) {
125
- scores[k_init] = 0.0;
127
+ scores[k_init] = f16(0.0);
126
128
  }
127
129
 
128
130
  for (var ht: u32 = 0u; ht < num_head_tiles; ht = ht + 1u) {
@@ -150,9 +152,9 @@ fn main(
150
152
  if (key_pos >= seq_len) { continue; }
151
153
  if (is_masked(query_pos, key_pos)) { continue; }
152
154
 
153
- var dot_partial: f32 = 0.0;
155
+ var dot_partial: f16 = f16(0.0);
154
156
  for (var td: u32 = 0u; td < tile_len; td = td + 1u) {
155
- dot_partial = dot_partial + q_local[d0 + td] * f32(shared_K[k * HEAD_TILE + td]);
157
+ dot_partial = dot_partial + q_local[d0 + td] * shared_K[k * HEAD_TILE + td];
156
158
  }
157
159
  scores[k] = scores[k] + dot_partial;
158
160
  }
@@ -161,17 +163,17 @@ fn main(
161
163
  workgroupBarrier();
162
164
  }
163
165
 
164
- var m_new: f32 = m_i;
166
+ var m_new: f16 = m_i;
165
167
  if (valid_query) {
166
- var block_max: f32 = -3.402823e+38;
168
+ var block_max: f16 = NEG_INF;
167
169
  for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
168
170
  let key_pos = kv_block_start + k;
169
171
  if (key_pos >= seq_len) { continue; }
170
172
  if (is_masked(query_pos, key_pos)) { continue; }
171
173
 
172
174
  var s = scores[k] * scale;
173
- if (u.attn_softcap > 0.0) {
174
- s = tanh(s / u.attn_softcap) * u.attn_softcap;
175
+ if (softcap > f16(0.0)) {
176
+ s = tanh(s / softcap) * softcap;
175
177
  }
176
178
  scores[k] = s;
177
179
  block_max = max(block_max, s);
@@ -217,7 +219,7 @@ fn main(
217
219
  }
218
220
 
219
221
  for (var td: u32 = 0u; td < tile_len; td = td + 1u) {
220
- acc[d0 + td] = acc[d0 + td] + p * f32(shared_V[k * HEAD_TILE + td]);
222
+ acc[d0 + td] = acc[d0 + td] + p * shared_V[k * HEAD_TILE + td];
221
223
  }
222
224
  }
223
225
  }
@@ -232,9 +234,9 @@ fn main(
232
234
 
233
235
  if (valid_query) {
234
236
  let out_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
235
- let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
237
+ let inv_l_i = select(f16(0.0), f16(1.0) / l_i, l_i > f16(0.0));
236
238
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
237
- output[out_offset + d] = f16(acc[d] * inv_l_i);
239
+ output[out_offset + d] = acc[d] * inv_l_i;
238
240
  }
239
241
  }
240
242
  }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_small.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused Multi-Head Attention Kernel (small tiles, f16 KV)
4
4
  //
5
5
  // Same algorithm as attention_small.wgsl but K/V are stored as f16.
@@ -1,10 +1,10 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_streaming.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
3
- // Streaming Multi-Head Attention Kernel (f16 QKV + f16 output)
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
+ // Streaming Multi-Head Attention Kernel (no workgroup storage)
4
4
  //
5
- // Same as attention_streaming.wgsl but Q/K/V are f16 and output is f16.
6
-
7
- enable f16;
5
+ // Fallback variant for devices with extremely small shared memory or
6
+ // models with head_dim beyond tiled support. Uses two-pass softmax and
7
+ // reads K/V directly from storage. Slower but compatible.
8
8
 
9
9
  override WORKGROUP_SIZE: u32 = 1u;
10
10
  const MAX_HEAD_DIM: u32 = 512u;
@@ -17,14 +17,18 @@ struct Uniforms {
17
17
  query_len: u32,
18
18
  scale: f32,
19
19
  is_causal: u32,
20
- start_pos: u32,
21
- attn_softcap: f32,
22
- sliding_window: u32,
23
- kv_len_source: u32,
20
+ start_pos: u32, // Absolute position offset for causal masking
21
+ attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
22
+ sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
23
+ kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
24
24
  kv_start: u32,
25
25
  page_size: u32,
26
26
  kv_layout: u32,
27
- _pad: u32,
27
+ bidirectional_span_start: u32,
28
+ bidirectional_span_length: u32,
29
+ _pad0: u32,
30
+ _pad1: u32,
31
+ _pad2: u32,
28
32
  }
29
33
 
30
34
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -40,10 +44,27 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
40
44
  return query_head_idx / heads_per_kv;
41
45
  }
42
46
 
47
+ fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
48
+ if (u.bidirectional_span_length == 0u) {
49
+ return false;
50
+ }
51
+ let span_start = u.bidirectional_span_start;
52
+ let span_end = span_start + u.bidirectional_span_length;
53
+ return abs_query >= span_start
54
+ && abs_query < span_end
55
+ && abs_key >= span_start
56
+ && abs_key < span_end;
57
+ }
58
+
43
59
  fn is_masked(query_pos: u32, key_pos: u32) -> bool {
44
60
  let abs_query = query_pos + u.start_pos;
45
61
  let abs_key = u.kv_start + key_pos;
46
- if (u.is_causal != 0u && abs_key > abs_query) { return true; }
62
+ // Causal mask
63
+ if (u.is_causal != 0u && abs_key > abs_query) {
64
+ if (is_bidirectional_span_visible(abs_query, abs_key)) { return false; }
65
+ return true;
66
+ }
67
+ // Sliding window mask
47
68
  if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
48
69
  if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
49
70
  }
@@ -99,9 +120,10 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>) {
99
120
  let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
100
121
  var dot: f32 = 0.0;
101
122
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
102
- dot = dot + q_local[d] * f32(K[k_offset + d]);
123
+ dot = dot + q_local[d] * K[k_offset + d];
103
124
  }
104
125
  dot = dot * scale;
126
+ // Gemma 2 attention softcapping
105
127
  if (u.attn_softcap > 0.0) {
106
128
  dot = tanh(dot / u.attn_softcap) * u.attn_softcap;
107
129
  }
@@ -121,27 +143,28 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>) {
121
143
  let v_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
122
144
  var dot: f32 = 0.0;
123
145
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
124
- dot = dot + q_local[d] * f32(K[k_offset + d]);
146
+ dot = dot + q_local[d] * K[k_offset + d];
125
147
  }
126
148
  dot = dot * scale;
149
+ // Gemma 2 attention softcapping
127
150
  if (u.attn_softcap > 0.0) {
128
151
  dot = tanh(dot / u.attn_softcap) * u.attn_softcap;
129
152
  }
130
153
  let w = exp(dot - max_score);
131
154
  sum_exp = sum_exp + w;
132
155
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
133
- acc[d] = acc[d] + w * f32(V[v_offset + d]);
156
+ acc[d] = acc[d] + w * V[v_offset + d];
134
157
  }
135
158
  }
136
159
 
137
160
  let out_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
138
161
  if (sum_exp <= 0.0) {
139
162
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
140
- output[out_offset + d] = f16(0.0);
163
+ output[out_offset + d] = 0.0;
141
164
  }
142
165
  return;
143
166
  }
144
167
  for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
145
- output[out_offset + d] = f16(acc[d] / sum_exp);
168
+ output[out_offset + d] = acc[d] / sum_exp;
146
169
  }
147
170
  }