doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,213 +0,0 @@
1
- --- src/gpu/kernels/attention_decode.wgsl
2
- +++ src/gpu/kernels/attention_decode_f16.wgsl
3
- @@ -1,28 +1,26 @@
4
- -// Fused Multi-Head Attention Kernel
5
- +// Fused Multi-Head Attention Kernel (decode, f16 QKV + f16 output)
6
- //
7
- -// Implements fused Q @ K^T → scale → mask → softmax → @ V
8
- -// Uses tiled/blocked approach to avoid materializing full attention matrix.
9
- -// Supports grouped query attention (GQA) where numKVHeads < numHeads.
10
- -//
11
- -// Based on Flash Attention principles adapted for WebGPU.
12
- +// Same as attention_decode_f16kv.wgsl but Q/K/V are f16 and output is f16.
13
- +
14
- +enable f16;
15
-
16
- const MAX_HEAD_DIM: u32 = 64u;
17
- const MAX_WORKGROUP_SIZE: u32 = 256u;
18
-
19
- -override DECODE_WORKGROUP_SIZE: u32 = 256u; // Decode kernel workgroup size (must match MAX_WORKGROUP_SIZE)
20
- +override DECODE_WORKGROUP_SIZE: u32 = 256u;
21
-
22
- struct Uniforms {
23
- - num_heads: u32, // Number of query heads
24
- - num_kv_heads: u32, // Number of KV heads (for GQA)
25
- - head_dim: u32, // Dimension per head
26
- - seq_len: u32, // Current sequence length (for KV)
27
- - query_len: u32, // Query length (1 for decode, seq_len for prefill)
28
- - scale: f32, // 1/sqrt(head_dim)
29
- - is_causal: u32, // Apply causal mask (1 = yes)
30
- - start_pos: u32, // Absolute position offset for causal masking
31
- - attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
32
- - sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
33
- - kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
34
- + num_heads: u32,
35
- + num_kv_heads: u32,
36
- + head_dim: u32,
37
- + seq_len: u32,
38
- + query_len: u32,
39
- + scale: f32,
40
- + is_causal: u32,
41
- + start_pos: u32,
42
- + attn_softcap: f32,
43
- + sliding_window: u32,
44
- + kv_len_source: u32,
45
- kv_start: u32,
46
- page_size: u32,
47
- kv_layout: u32,
48
- @@ -30,44 +28,28 @@
49
- }
50
-
51
- @group(0) @binding(0) var<uniform> u: Uniforms;
52
- -@group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
53
- -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
54
- -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
55
- -@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
56
- +@group(0) @binding(1) var<storage, read> Q: array<f16>;
57
- +@group(0) @binding(2) var<storage, read> K: array<f16>;
58
- +@group(0) @binding(3) var<storage, read> V: array<f16>;
59
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>;
60
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
61
- @group(0) @binding(6) var<storage, read> page_table: array<u32>;
62
-
63
- -// Online softmax accumulators (per-thread)
64
- -// Sized for 256 to support attention_decode workgroup size
65
- var<workgroup> row_max: array<f32, MAX_WORKGROUP_SIZE>;
66
- var<workgroup> row_sum: array<f32, MAX_WORKGROUP_SIZE>;
67
-
68
- -// Get KV head index for grouped query attention
69
- fn get_kv_head_idx(query_head_idx: u32) -> u32 {
70
- - // GQA: multiple query heads share one KV head
71
- let heads_per_kv = u.num_heads / u.num_kv_heads;
72
- return query_head_idx / heads_per_kv;
73
- }
74
-
75
- -// Check if position should be masked (causal + sliding window attention)
76
- fn is_masked(query_pos: u32, key_pos: u32) -> bool {
77
- - // Compute absolute positions
78
- let abs_query = query_pos + u.start_pos;
79
- let abs_key = u.kv_start + key_pos;
80
- -
81
- - // Causal mask: query can only attend to keys at same or earlier positions
82
- - if (u.is_causal != 0u && abs_key > abs_query) {
83
- - return true;
84
- - }
85
- -
86
- - // Sliding window mask: query can only attend to keys within the window
87
- - // Key must be >= (query - window + 1), i.e., within the last `window` positions
88
- + if (u.is_causal != 0u && abs_key > abs_query) { return true; }
89
- if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
90
- - if (abs_key < abs_query - u.sliding_window + 1u) {
91
- - return true; // Key is too far in the past
92
- - }
93
- - }
94
- -
95
- + if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
96
- + }
97
- return false;
98
- }
99
-
100
- @@ -92,12 +74,6 @@
101
- return kv_len_buffer[0];
102
- }
103
-
104
- -// Main attention kernel - one workgroup per (query_block, head)
105
- -// Workgroups are dispatched linearly as: numQueryBlocks * numHeads.
106
- -// head_idx and query_block_idx are derived from workgroup_id.x.
107
- -
108
- -// Simplified single-query attention for decode step
109
- -// More efficient when query_len == 1
110
- @compute @workgroup_size(DECODE_WORKGROUP_SIZE, 1, 1)
111
- fn attention_decode(
112
- @builtin(global_invocation_id) global_id: vec3<u32>,
113
- @@ -119,17 +95,14 @@
114
- let seq_len = get_kv_len();
115
- let scale = u.scale;
116
-
117
- - // Each thread handles a subset of key positions
118
- let keys_per_thread = (seq_len + DECODE_WORKGROUP_SIZE - 1u) / DECODE_WORKGROUP_SIZE;
119
-
120
- - // Load query (single position)
121
- var q_local: array<f32, MAX_HEAD_DIM>;
122
- let q_offset = head_idx * head_dim;
123
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
124
- - q_local[d] = Q[q_offset + d];
125
- - }
126
- -
127
- - // Pass 1: local max
128
- + q_local[d] = f32(Q[q_offset + d]);
129
- + }
130
- +
131
- var local_max: f32 = -3.402823e+38;
132
- let query_pos = 0u;
133
-
134
- @@ -143,11 +116,10 @@
135
-
136
- var score: f32 = 0.0;
137
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
138
- - score = score + q_local[d] * K[k_offset + d];
139
- + score = score + q_local[d] * f32(K[k_offset + d]);
140
- }
141
- score = score * scale;
142
-
143
- - // Gemma 2 attention softcapping
144
- if (u.attn_softcap > 0.0) {
145
- score = tanh(score / u.attn_softcap) * u.attn_softcap;
146
- }
147
- @@ -155,11 +127,9 @@
148
- local_max = max(local_max, score);
149
- }
150
-
151
- - // Store local max for reduction
152
- row_max[thread_idx] = local_max;
153
- workgroupBarrier();
154
-
155
- - // Parallel reduction to find global max
156
- for (var stride: u32 = DECODE_WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
157
- if (thread_idx < stride && thread_idx + stride < DECODE_WORKGROUP_SIZE) {
158
- row_max[thread_idx] = max(row_max[thread_idx], row_max[thread_idx + stride]);
159
- @@ -169,7 +139,6 @@
160
-
161
- let global_max = row_max[0];
162
-
163
- - // Pass 2: local sum and output accumulator
164
- var local_sum: f32 = 0.0;
165
- var local_out: array<f32, MAX_HEAD_DIM>;
166
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
167
- @@ -187,7 +156,7 @@
168
-
169
- var score: f32 = 0.0;
170
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
171
- - score = score + q_local[d] * K[k_offset + d];
172
- + score = score + q_local[d] * f32(K[k_offset + d]);
173
- }
174
- score = score * scale;
175
-
176
- @@ -198,14 +167,13 @@
177
- let exp_score = exp(score - global_max);
178
- local_sum = local_sum + exp_score;
179
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
180
- - local_out[d] = local_out[d] + exp_score * V[v_offset + d];
181
- + local_out[d] = local_out[d] + exp_score * f32(V[v_offset + d]);
182
- }
183
- }
184
-
185
- row_sum[thread_idx] = local_sum;
186
- workgroupBarrier();
187
-
188
- - // Parallel reduction for sum
189
- for (var stride: u32 = DECODE_WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
190
- if (thread_idx < stride && thread_idx + stride < DECODE_WORKGROUP_SIZE) {
191
- row_sum[thread_idx] = row_sum[thread_idx] + row_sum[thread_idx + stride];
192
- @@ -216,7 +184,6 @@
193
- let global_sum = row_sum[0];
194
- let inv_sum = select(0.0, 1.0 / global_sum, global_sum > 0.0);
195
-
196
- - // Reduction for output (per-dimension)
197
- let out_offset = head_idx * head_dim;
198
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
199
- row_sum[thread_idx] = local_out[d];
200
- @@ -226,8 +193,8 @@
201
- for (var t: u32 = 0u; t < DECODE_WORKGROUP_SIZE; t = t + 1u) {
202
- sum = sum + row_sum[t];
203
- }
204
- - output[out_offset + d] = sum * inv_sum;
205
- - }
206
- - workgroupBarrier();
207
- - }
208
- -}
209
- + output[out_offset + d] = f16(sum * inv_sum);
210
- + }
211
- + workgroupBarrier();
212
- + }
213
- +}
@@ -1,89 +0,0 @@
1
- --- src/gpu/kernels/attention_decode.wgsl
2
- +++ src/gpu/kernels/attention_decode_f16kv.wgsl
3
- @@ -1,10 +1,9 @@
4
- -// Fused Multi-Head Attention Kernel
5
- +// Fused Multi-Head Attention Kernel (f16 KV)
6
- //
7
- -// Implements fused Q @ K^T → scale → mask → softmax → @ V
8
- -// Uses tiled/blocked approach to avoid materializing full attention matrix.
9
- -// Supports grouped query attention (GQA) where numKVHeads < numHeads.
10
- -//
11
- -// Based on Flash Attention principles adapted for WebGPU.
12
- +// Same as attention.wgsl but K/V are stored as f16 for KV-cache compression.
13
- +// Q and output remain f32; K/V are cast to f32 on load.
14
- +
15
- +enable f16;
16
-
17
- const MAX_HEAD_DIM: u32 = 64u;
18
- const MAX_WORKGROUP_SIZE: u32 = 256u;
19
- @@ -31,8 +30,8 @@
20
-
21
- @group(0) @binding(0) var<uniform> u: Uniforms;
22
- @group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
23
- -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
24
- -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
25
- +@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
26
- +@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
27
- @group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
28
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
29
- @group(0) @binding(6) var<storage, read> page_table: array<u32>;
30
- @@ -51,23 +50,14 @@
31
-
32
- // Check if position should be masked (causal + sliding window attention)
33
- fn is_masked(query_pos: u32, key_pos: u32) -> bool {
34
- - // Compute absolute positions
35
- let abs_query = query_pos + u.start_pos;
36
- let abs_key = u.kv_start + key_pos;
37
- -
38
- - // Causal mask: query can only attend to keys at same or earlier positions
39
- - if (u.is_causal != 0u && abs_key > abs_query) {
40
- - return true;
41
- - }
42
- -
43
- - // Sliding window mask: query can only attend to keys within the window
44
- - // Key must be >= (query - window + 1), i.e., within the last `window` positions
45
- + // Causal mask
46
- + if (u.is_causal != 0u && abs_key > abs_query) { return true; }
47
- + // Sliding window mask
48
- if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
49
- - if (abs_key < abs_query - u.sliding_window + 1u) {
50
- - return true; // Key is too far in the past
51
- - }
52
- - }
53
- -
54
- + if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
55
- + }
56
- return false;
57
- }
58
-
59
- @@ -143,11 +133,10 @@
60
-
61
- var score: f32 = 0.0;
62
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
63
- - score = score + q_local[d] * K[k_offset + d];
64
- + score = score + q_local[d] * f32(K[k_offset + d]);
65
- }
66
- score = score * scale;
67
-
68
- - // Gemma 2 attention softcapping
69
- if (u.attn_softcap > 0.0) {
70
- score = tanh(score / u.attn_softcap) * u.attn_softcap;
71
- }
72
- @@ -187,7 +176,7 @@
73
-
74
- var score: f32 = 0.0;
75
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
76
- - score = score + q_local[d] * K[k_offset + d];
77
- + score = score + q_local[d] * f32(K[k_offset + d]);
78
- }
79
- score = score * scale;
80
-
81
- @@ -198,7 +187,7 @@
82
- let exp_score = exp(score - global_max);
83
- local_sum = local_sum + exp_score;
84
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
85
- - local_out[d] = local_out[d] + exp_score * V[v_offset + d];
86
- + local_out[d] = local_out[d] + exp_score * f32(V[v_offset + d]);
87
- }
88
- }
89
-
@@ -1,62 +0,0 @@
1
- --- src/gpu/kernels/attention.wgsl
2
- +++ src/gpu/kernels/attention_f16.wgsl
3
- @@ -6,6 +6,8 @@
4
- //
5
- // Based on Flash Attention principles adapted for WebGPU.
6
-
7
- +enable f16;
8
- +
9
- // Tile sizes for blocked attention
10
- const MAX_BLOCK_SIZE: u32 = 32u;
11
- const MAX_HEAD_TILE: u32 = 64u;
12
- @@ -38,10 +40,10 @@
13
- }
14
-
15
- @group(0) @binding(0) var<uniform> u: Uniforms;
16
- -@group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
17
- -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
18
- -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
19
- -@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
20
- +@group(0) @binding(1) var<storage, read> Q: array<f16>; // [query_len, num_heads, head_dim]
21
- +@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
22
- +@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
23
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>; // [query_len, num_heads, head_dim]
24
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
25
- @group(0) @binding(6) var<storage, read> page_table: array<u32>;
26
-
27
- @@ -156,7 +158,7 @@
28
- if (valid_query) {
29
- let q_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
30
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
31
- - q_local[d] = Q[q_offset + d];
32
- + q_local[d] = f32(Q[q_offset + d]);
33
- }
34
- }
35
-
36
- @@ -172,7 +174,7 @@
37
- let k_idx = get_kv_pos(k_load_idx);
38
- let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
39
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
40
- - shared_K[thread_idx * head_dim + d] = K[k_offset + d];
41
- + shared_K[thread_idx * head_dim + d] = f32(K[k_offset + d]);
42
- }
43
- } else {
44
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
45
- @@ -186,7 +188,7 @@
46
- let v_idx = get_kv_pos(v_load_idx);
47
- let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
48
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
49
- - shared_V[thread_idx * head_dim + d] = V[v_offset + d];
50
- + shared_V[thread_idx * head_dim + d] = f32(V[v_offset + d]);
51
- }
52
- } else {
53
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
54
- @@ -262,7 +264,7 @@
55
- let out_offset = query_pos * u.num_heads * head_dim + head_idx * head_dim;
56
- let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
57
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
58
- - output[out_offset + d] = acc[d] * inv_l_i;
59
- + output[out_offset + d] = f16(acc[d] * inv_l_i);
60
- }
61
- }
62
- }
@@ -1,40 +0,0 @@
1
- --- src/gpu/kernels/attention.wgsl
2
- +++ src/gpu/kernels/attention_f16kv.wgsl
3
- @@ -6,6 +6,8 @@
4
- //
5
- // Based on Flash Attention principles adapted for WebGPU.
6
-
7
- +enable f16;
8
- +
9
- // Tile sizes for blocked attention
10
- const MAX_BLOCK_SIZE: u32 = 32u;
11
- const MAX_HEAD_TILE: u32 = 64u;
12
- @@ -39,8 +41,8 @@
13
-
14
- @group(0) @binding(0) var<uniform> u: Uniforms;
15
- @group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
16
- -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
17
- -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
18
- +@group(0) @binding(2) var<storage, read> K: array<f16>; // [seq_len, num_kv_heads, head_dim]
19
- +@group(0) @binding(3) var<storage, read> V: array<f16>; // [seq_len, num_kv_heads, head_dim]
20
- @group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
21
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
22
- @group(0) @binding(6) var<storage, read> page_table: array<u32>;
23
- @@ -172,7 +174,7 @@
24
- let k_idx = get_kv_pos(k_load_idx);
25
- let k_offset = k_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
26
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
27
- - shared_K[thread_idx * head_dim + d] = K[k_offset + d];
28
- + shared_K[thread_idx * head_dim + d] = f32(K[k_offset + d]);
29
- }
30
- } else {
31
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
32
- @@ -186,7 +188,7 @@
33
- let v_idx = get_kv_pos(v_load_idx);
34
- let v_offset = v_idx * u.num_kv_heads * head_dim + kv_head_idx * head_dim;
35
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
36
- - shared_V[thread_idx * head_dim + d] = V[v_offset + d];
37
- + shared_V[thread_idx * head_dim + d] = f32(V[v_offset + d]);
38
- }
39
- } else {
40
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
@@ -1,226 +0,0 @@
1
- --- src/gpu/kernels/attention_small.wgsl
2
- +++ src/gpu/kernels/attention_small_f16.wgsl
3
- @@ -1,39 +1,28 @@
4
- -// Fused Multi-Head Attention Kernel (small tiles)
5
- +// Fused Multi-Head Attention Kernel (small tiles, f16 QKV + f16 output)
6
- //
7
- -// Compatibility-focused variant for devices with limited workgroup storage
8
- -// and for models with larger head_dim (up to MAX_HEAD_DIM).
9
- -//
10
- -// Differences vs attention.wgsl:
11
- -// - BLOCK_SIZE = 32, HEAD_TILE = 32
12
- -// - No shared score matrix (recompute in second pass)
13
- -// - Tiles over head_dim to support large heads
14
- -//
15
- -// Q and output are f32. K/V are f32.
16
- -//
17
- -// One workgroup per (query_block, head) encoded linearly in workgroup_id.x:
18
- -// linear = workgroup_id.x
19
- -// head_idx = linear % num_heads
20
- -// query_block_idx = linear / num_heads
21
- -//
22
- -// workgroup_size = BLOCK_SIZE (one thread per query position in the block).
23
- +// Same algorithm as attention_small.wgsl but Q/K/V are f16 and output is f16.
24
- +// Shared tiles and attention arithmetic use f16 for the all-f16 lane.
25
-
26
- -const BLOCK_SIZE: u32 = 32u; // Sequence tile size
27
- -const HEAD_TILE: u32 = 32u; // Head dimension tile
28
- -const MAX_HEAD_DIM: u32 = 256u;
29
- +enable f16;
30
- +
31
- +const BLOCK_SIZE: u32 = 32u;
32
- const WORKGROUP_SIZE: u32 = BLOCK_SIZE;
33
- +const HEAD_TILE: u32 = 32u;
34
- +const MAX_HEAD_DIM: u32 = 256u;
35
- +const NEG_INF = f16(-65504.0);
36
-
37
- struct Uniforms {
38
- - num_heads: u32, // Number of query heads
39
- - num_kv_heads: u32, // Number of KV heads (for GQA)
40
- - head_dim: u32, // Dimension per head
41
- - seq_len: u32, // KV length
42
- - query_len: u32, // Query length (1 for decode, >1 for prefill)
43
- - scale: f32, // 1/sqrt(head_dim)
44
- - is_causal: u32, // Apply causal mask (1 = yes)
45
- - start_pos: u32, // Absolute position offset for causal masking
46
- - attn_softcap: f32, // Gemma 2: 50.0, 0 = disabled
47
- - sliding_window: u32, // Sliding window size (0 = disabled, >0 = window size)
48
- - kv_len_source: u32, // 0 = use uniform seq_len, 1 = use buffer
49
- + num_heads: u32,
50
- + num_kv_heads: u32,
51
- + head_dim: u32,
52
- + seq_len: u32,
53
- + query_len: u32,
54
- + scale: f32,
55
- + is_causal: u32,
56
- + start_pos: u32,
57
- + attn_softcap: f32,
58
- + sliding_window: u32,
59
- + kv_len_source: u32,
60
- kv_start: u32,
61
- page_size: u32,
62
- kv_layout: u32,
63
- @@ -41,16 +30,15 @@
64
- }
65
-
66
- @group(0) @binding(0) var<uniform> u: Uniforms;
67
- -@group(0) @binding(1) var<storage, read> Q: array<f32>; // [query_len, num_heads, head_dim]
68
- -@group(0) @binding(2) var<storage, read> K: array<f32>; // [seq_len, num_kv_heads, head_dim]
69
- -@group(0) @binding(3) var<storage, read> V: array<f32>; // [seq_len, num_kv_heads, head_dim]
70
- -@group(0) @binding(4) var<storage, read_write> output: array<f32>; // [query_len, num_heads, head_dim]
71
- +@group(0) @binding(1) var<storage, read> Q: array<f16>;
72
- +@group(0) @binding(2) var<storage, read> K: array<f16>;
73
- +@group(0) @binding(3) var<storage, read> V: array<f16>;
74
- +@group(0) @binding(4) var<storage, read_write> output: array<f16>;
75
- @group(0) @binding(5) var<storage, read> kv_len_buffer: array<u32>;
76
- @group(0) @binding(6) var<storage, read> page_table: array<u32>;
77
-
78
- -// Shared memory for tiled K/V slices
79
- -var<workgroup> shared_K: array<f32, BLOCK_SIZE * HEAD_TILE>;
80
- -var<workgroup> shared_V: array<f32, BLOCK_SIZE * HEAD_TILE>;
81
- +var<workgroup> shared_K: array<f16, BLOCK_SIZE * HEAD_TILE>;
82
- +var<workgroup> shared_V: array<f16, BLOCK_SIZE * HEAD_TILE>;
83
-
84
- fn get_kv_head_idx(query_head_idx: u32) -> u32 {
85
- let heads_per_kv = u.num_heads / u.num_kv_heads;
86
- @@ -60,9 +48,7 @@
87
- fn is_masked(query_pos: u32, key_pos: u32) -> bool {
88
- let abs_query = query_pos + u.start_pos;
89
- let abs_key = u.kv_start + key_pos;
90
- - // Causal mask
91
- if (u.is_causal != 0u && abs_key > abs_query) { return true; }
92
- - // Sliding window mask
93
- if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
94
- if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
95
- }
96
- @@ -108,24 +94,25 @@
97
- }
98
- let seq_len = get_kv_len();
99
- let query_len = u.query_len;
100
- - let scale = u.scale;
101
- + let scale = f16(u.scale);
102
- + let softcap = f16(u.attn_softcap);
103
-
104
- let query_pos = query_block_idx * BLOCK_SIZE + thread_idx;
105
- let valid_query = query_pos < query_len;
106
-
107
- - var q_local: array<f32, MAX_HEAD_DIM>;
108
- - var acc: array<f32, MAX_HEAD_DIM>;
109
- + var q_local: array<f16, MAX_HEAD_DIM>;
110
- + var acc: array<f16, MAX_HEAD_DIM>;
111
-
112
- if (valid_query) {
113
- let q_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
114
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
115
- q_local[d] = Q[q_offset + d];
116
- - acc[d] = 0.0;
117
- + acc[d] = f16(0.0);
118
- }
119
- }
120
-
121
- - var m_i: f32 = -3.402823e+38;
122
- - var l_i: f32 = 0.0;
123
- + var m_i: f16 = NEG_INF;
124
- + var l_i: f16 = f16(0.0);
125
-
126
- let num_kv_blocks = (seq_len + BLOCK_SIZE - 1u) / BLOCK_SIZE;
127
- let num_head_tiles = (head_dim + HEAD_TILE - 1u) / HEAD_TILE;
128
- @@ -133,17 +120,15 @@
129
- for (var kv_block: u32 = 0u; kv_block < num_kv_blocks; kv_block = kv_block + 1u) {
130
- let kv_block_start = kv_block * BLOCK_SIZE;
131
-
132
- - var scores: array<f32, BLOCK_SIZE>;
133
- + var scores: array<f16, BLOCK_SIZE>;
134
- for (var k_init: u32 = 0u; k_init < BLOCK_SIZE; k_init = k_init + 1u) {
135
- - scores[k_init] = 0.0;
136
- + scores[k_init] = f16(0.0);
137
- }
138
-
139
- - // Compute dot products for this KV block by tiling head_dim.
140
- for (var ht: u32 = 0u; ht < num_head_tiles; ht = ht + 1u) {
141
- let d0 = ht * HEAD_TILE;
142
- let tile_len = min(HEAD_TILE, head_dim - d0);
143
-
144
- - // Load K slice for this block into shared memory.
145
- let key_pos_load = kv_block_start + thread_idx;
146
- if (key_pos_load < seq_len) {
147
- let k_idx = get_kv_pos(key_pos_load);
148
- @@ -153,7 +138,7 @@
149
- }
150
- } else {
151
- for (var td: u32 = 0u; td < tile_len; td = td + 1u) {
152
- - shared_K[thread_idx * HEAD_TILE + td] = 0.0;
153
- + shared_K[thread_idx * HEAD_TILE + td] = f16(0.0);
154
- }
155
- }
156
-
157
- @@ -165,7 +150,7 @@
158
- if (key_pos >= seq_len) { continue; }
159
- if (is_masked(query_pos, key_pos)) { continue; }
160
-
161
- - var dot_partial: f32 = 0.0;
162
- + var dot_partial: f16 = f16(0.0);
163
- for (var td: u32 = 0u; td < tile_len; td = td + 1u) {
164
- dot_partial = dot_partial + q_local[d0 + td] * shared_K[k * HEAD_TILE + td];
165
- }
166
- @@ -176,18 +161,17 @@
167
- workgroupBarrier();
168
- }
169
-
170
- - var m_new: f32 = m_i;
171
- + var m_new: f16 = m_i;
172
- if (valid_query) {
173
- - var block_max: f32 = -3.402823e+38;
174
- + var block_max: f16 = NEG_INF;
175
- for (var k: u32 = 0u; k < BLOCK_SIZE; k = k + 1u) {
176
- let key_pos = kv_block_start + k;
177
- if (key_pos >= seq_len) { continue; }
178
- if (is_masked(query_pos, key_pos)) { continue; }
179
-
180
- var s = scores[k] * scale;
181
- - // Gemma 2 attention softcapping
182
- - if (u.attn_softcap > 0.0) {
183
- - s = tanh(s / u.attn_softcap) * u.attn_softcap;
184
- + if (softcap > f16(0.0)) {
185
- + s = tanh(s / softcap) * softcap;
186
- }
187
- scores[k] = s;
188
- block_max = max(block_max, s);
189
- @@ -202,8 +186,6 @@
190
- }
191
- }
192
-
193
- - // Accumulate V contribution by tiling head_dim again.
194
- - // Barriers must be uniform, so only math is guarded by valid_query.
195
- for (var ht: u32 = 0u; ht < num_head_tiles; ht = ht + 1u) {
196
- let d0 = ht * HEAD_TILE;
197
- let tile_len = min(HEAD_TILE, head_dim - d0);
198
- @@ -217,7 +199,7 @@
199
- }
200
- } else {
201
- for (var td: u32 = 0u; td < tile_len; td = td + 1u) {
202
- - shared_V[thread_idx * HEAD_TILE + td] = 0.0;
203
- + shared_V[thread_idx * HEAD_TILE + td] = f16(0.0);
204
- }
205
- }
206
-
207
- @@ -230,7 +212,6 @@
208
- if (is_masked(query_pos, key_pos)) { continue; }
209
-
210
- let p = exp(scores[k] - m_new);
211
- - // Only accumulate l_i on first head tile to avoid double counting
212
- if (ht == 0u) {
213
- l_i = l_i + p;
214
- }
215
- @@ -249,10 +230,9 @@
216
- }
217
- }
218
-
219
- - // Always write output when valid_query - write zeros if l_i is 0 to avoid garbage
220
- if (valid_query) {
221
- let out_offset = query_pos * num_heads * head_dim + head_idx * head_dim;
222
- - let inv_l_i = select(0.0, 1.0 / l_i, l_i > 0.0);
223
- + let inv_l_i = select(f16(0.0), f16(1.0) / l_i, l_i > f16(0.0));
224
- for (var d: u32 = 0u; d < head_dim; d = d + 1u) {
225
- output[out_offset + d] = acc[d] * inv_l_i;
226
- }