doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,124 @@
1
+ // Fused Gate + Up Projection + GeGLU Activation Kernel (F16, gated)
2
+ //
3
+ // Fuses the two FFN projection matmuls with the GeGLU activation:
4
+ // gate[m, n] = sum_k(input[m, k] * W_gate[k, n])
5
+ // up[m, n] = sum_k(input[m, k] * W_up[k, n])
6
+ // output[m, n] = gelu_tanh(gate[m, n]) * up[m, n]
7
+ //
8
+ // Each workgroup owns one row m and a column tile of the activation. Threads
9
+ // in a column group cooperate on BOTH gate and up dot products — the input
10
+ // row is read once and each loaded element feeds both matmuls, halving input
11
+ // memory traffic vs two separate fused matmul dispatches.
12
+ //
13
+ // The activation is GeLU-tanh applied to the gate branch, then multiplied by
14
+ // the up branch (GeGLU). Silu variants exist in `fused_ffn_q4k_f16.wgsl`; this
15
+ // variant ships gelu specifically for gated-gelu transformer families and has
16
+ // no runtime-selectable activation (keeping the inner loop branch-free).
17
+ //
18
+ // `intermediate_size` is a uniform — per-layer variation (e.g., double-wide
19
+ // KV-shared layers) is handled without kernel change.
20
+ //
21
+ // Dispatch: (ceil(intermediate_size / COLS_PER_WG), M, 1)
22
+ //
23
+ // Accumulation in f32 for numerical stability; output is f16 to match the
24
+ // downstream down_proj contract.
25
+ //
26
+ // No thread returns on a divergent condition mid-kernel.
27
+
28
+ enable f16;
29
+
30
+ override WORKGROUP_SIZE: u32 = 128u;
31
+ override COLS_PER_WG: u32 = 8u;
32
+
33
+ const MAX_WG: u32 = 256u;
34
+
35
+ struct Uniforms {
36
+ M: u32,
37
+ hidden_size: u32,
38
+ intermediate_size: u32,
39
+ transpose_b: u32,
40
+ _pad0: u32,
41
+ _pad1: u32,
42
+ _pad2: u32,
43
+ _pad3: u32,
44
+ }
45
+
46
+ @group(0) @binding(0) var<uniform> u: Uniforms;
47
+ @group(0) @binding(1) var<storage, read> input: array<f16>; // [M, hidden_size]
48
+ @group(0) @binding(2) var<storage, read> W_gate: array<f16>; // [hidden, inter] or [inter, hidden]
49
+ @group(0) @binding(3) var<storage, read> W_up: array<f16>; // same layout as W_gate
50
+ @group(0) @binding(4) var<storage, read_write> output: array<f16>; // [M, intermediate_size]
51
+
52
+ var<workgroup> shared_gate: array<f32, MAX_WG>;
53
+ var<workgroup> shared_up: array<f32, MAX_WG>;
54
+
55
+ fn gelu_tanh(x: f32) -> f32 {
56
+ // GELU with tanh approximation — matches pytorch gelu_approx='tanh' and
57
+ // Gemma's gelu_pytorch_tanh activation.
58
+ let c: f32 = 0.7978845608;
59
+ let k: f32 = 0.044715;
60
+ return 0.5 * x * (1.0 + tanh(c * (x + k * x * x * x)));
61
+ }
62
+
63
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
64
+ fn main(
65
+ @builtin(workgroup_id) wg: vec3<u32>,
66
+ @builtin(local_invocation_id) lid: vec3<u32>
67
+ ) {
68
+ let col_tile = wg.x;
69
+ let row = wg.y;
70
+ let tid = lid.x;
71
+ if (row >= u.M) { return; }
72
+
73
+ let K = u.hidden_size;
74
+ let N = u.intermediate_size;
75
+
76
+ let threads_per_col = WORKGROUP_SIZE / COLS_PER_WG;
77
+ let col_within_tile = tid / threads_per_col;
78
+ let thread_in_col = tid % threads_per_col;
79
+ let col = col_tile * COLS_PER_WG + col_within_tile;
80
+ let col_valid = col < N;
81
+
82
+ var gate_partial: f32 = 0.0;
83
+ var up_partial: f32 = 0.0;
84
+ if (col_valid) {
85
+ let input_row_base = row * K;
86
+ if (u.transpose_b != 0u) {
87
+ let w_base = col * K;
88
+ for (var k: u32 = thread_in_col; k < K; k = k + threads_per_col) {
89
+ let a = f32(input[input_row_base + k]);
90
+ gate_partial = gate_partial + a * f32(W_gate[w_base + k]);
91
+ up_partial = up_partial + a * f32(W_up[w_base + k]);
92
+ }
93
+ } else {
94
+ for (var k: u32 = thread_in_col; k < K; k = k + threads_per_col) {
95
+ let a = f32(input[input_row_base + k]);
96
+ gate_partial = gate_partial + a * f32(W_gate[k * N + col]);
97
+ up_partial = up_partial + a * f32(W_up[k * N + col]);
98
+ }
99
+ }
100
+ }
101
+
102
+ shared_gate[tid] = gate_partial;
103
+ shared_up[tid] = up_partial;
104
+ workgroupBarrier();
105
+
106
+ var stride: u32 = threads_per_col >> 1u;
107
+ loop {
108
+ if (stride == 0u) { break; }
109
+ if (thread_in_col < stride) {
110
+ let lane = col_within_tile * threads_per_col + thread_in_col;
111
+ shared_gate[lane] = shared_gate[lane] + shared_gate[lane + stride];
112
+ shared_up[lane] = shared_up[lane] + shared_up[lane + stride];
113
+ }
114
+ workgroupBarrier();
115
+ stride = stride >> 1u;
116
+ }
117
+
118
+ if (thread_in_col == 0u && col_valid) {
119
+ let lane = col_within_tile * threads_per_col;
120
+ let gate_val = shared_gate[lane];
121
+ let up_val = shared_up[lane];
122
+ output[row * N + col] = f16(gelu_tanh(gate_val) * up_val);
123
+ }
124
+ }
@@ -15,12 +15,11 @@ enable subgroups;
15
15
 
16
16
  // Q4_K constants
17
17
  const QK_K: u32 = 256u; // Elements per super-block
18
- const BLOCK_SIZE: u32 = 144u; // Bytes per Q4_K block
19
18
  const SUBBLOCK_SIZE: u32 = 32u; // Elements per sub-block
19
+ const NUM_SUBBLOCKS: u32 = QK_K / SUBBLOCK_SIZE; // 8 sub-blocks per super-block
20
20
 
21
21
  override WORKGROUP_SIZE: u32 = 256u;
22
22
  const MAX_WORKGROUP_SIZE: u32 = 256u;
23
- const MAX_SUBGROUPS: u32 = 256u; // Supports subgroup_size >= 1
24
23
 
25
24
  struct Uniforms {
26
25
  M: u32, // Always 1 for GEMV
@@ -46,9 +45,6 @@ struct Q4KBlock {
46
45
  @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
47
46
  @group(0) @binding(3) var<storage, read_write> C: array<f32>;
48
47
 
49
- // Shared memory for subgroup reduction
50
- var<workgroup> wg_sums: array<f32, MAX_SUBGROUPS>;
51
-
52
48
  // Extract f16 from packed u32
53
49
  fn unpack_f16_lo(packed: u32) -> f32 {
54
50
  return unpack2x16float(packed).x;
@@ -102,153 +98,6 @@ fn get_q4(qs: array<u32, 32>, idx: u32) -> u32 {
102
98
  }
103
99
  }
104
100
 
105
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
106
- fn main(
107
- @builtin(local_invocation_id) lid: vec3<u32>,
108
- @builtin(workgroup_id) wg_id: vec3<u32>,
109
- @builtin(subgroup_invocation_id) sg_id: u32,
110
- @builtin(subgroup_size) sg_size: u32
111
- ) {
112
- let col = wg_id.x; // Output column this workgroup computes
113
- let local_id = lid.x;
114
-
115
- // Track validity - NO early return to maintain uniform control flow for subgroupAdd
116
- let is_valid = col < u.N;
117
-
118
- var partial_sum: f32 = 0.0;
119
-
120
- // Only do work if this column is valid
121
- if (is_valid) {
122
- // Each thread processes some Q4_K blocks
123
- let num_blocks = u.num_blocks_per_row;
124
- let tail_size = u.K & 255u;
125
- let full_blocks = num_blocks - select(0u, 1u, tail_size > 0u);
126
- let blocks_per_thread = (num_blocks + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
127
- let block_start = local_id * blocks_per_thread;
128
- let block_end = min(block_start + blocks_per_thread, num_blocks);
129
- let full_end = min(block_end, full_blocks);
130
-
131
- // B_q4k layout: row-major [N, K/256] - block b for column col is at col * num_blocks + b
132
- for (var b: u32 = block_start; b < full_end; b = b + 1u) {
133
- let block = B_q4k[col * num_blocks + b];
134
- let d = unpack_f16_lo(block.d_dmin);
135
- let dmin = unpack_f16_hi(block.d_dmin);
136
- let k_base = b * QK_K;
137
-
138
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
139
- let sm = get_scale_min_k4(block.scales, sb);
140
- let scale = d * f32(sm.x);
141
- let min_val = dmin * f32(sm.y);
142
-
143
- let sb_base = sb * SUBBLOCK_SIZE;
144
-
145
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
146
- let elem0 = sb_base + i;
147
- let elem1 = sb_base + i + 1u;
148
- let elem2 = sb_base + i + 2u;
149
- let elem3 = sb_base + i + 3u;
150
-
151
- let k0 = k_base + elem0;
152
- let k1 = k_base + elem1;
153
- let k2 = k_base + elem2;
154
- let k3 = k_base + elem3;
155
-
156
- let a0 = A[k0];
157
- let a1 = A[k1];
158
- let a2 = A[k2];
159
- let a3 = A[k3];
160
-
161
- let q0 = get_q4(block.qs, elem0);
162
- let q1 = get_q4(block.qs, elem1);
163
- let q2 = get_q4(block.qs, elem2);
164
- let q3 = get_q4(block.qs, elem3);
165
-
166
- let w0 = scale * f32(q0) - min_val;
167
- let w1 = scale * f32(q1) - min_val;
168
- let w2 = scale * f32(q2) - min_val;
169
- let w3 = scale * f32(q3) - min_val;
170
-
171
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
172
- }
173
- }
174
- }
175
-
176
- if (tail_size > 0u) {
177
- let tail_block = full_blocks;
178
- if (tail_block >= block_start && tail_block < block_end) {
179
- let block = B_q4k[col * num_blocks + tail_block];
180
- let d = unpack_f16_lo(block.d_dmin);
181
- let dmin = unpack_f16_hi(block.d_dmin);
182
- let k_base = tail_block * QK_K;
183
-
184
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
185
- let sb_base = sb * SUBBLOCK_SIZE;
186
- if (sb_base >= tail_size) {
187
- break;
188
- }
189
- let sm = get_scale_min_k4(block.scales, sb);
190
- let scale = d * f32(sm.x);
191
- let min_val = dmin * f32(sm.y);
192
-
193
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
194
- let elem0 = sb_base + i;
195
- let elem1 = sb_base + i + 1u;
196
- let elem2 = sb_base + i + 2u;
197
- let elem3 = sb_base + i + 3u;
198
-
199
- let k0 = k_base + elem0;
200
- let k1 = k_base + elem1;
201
- let k2 = k_base + elem2;
202
- let k3 = k_base + elem3;
203
-
204
- var a0: f32 = 0.0;
205
- var a1: f32 = 0.0;
206
- var a2: f32 = 0.0;
207
- var a3: f32 = 0.0;
208
- if (k0 < u.K) { a0 = A[k0]; }
209
- if (k1 < u.K) { a1 = A[k1]; }
210
- if (k2 < u.K) { a2 = A[k2]; }
211
- if (k3 < u.K) { a3 = A[k3]; }
212
-
213
- let q0 = get_q4(block.qs, elem0);
214
- let q1 = get_q4(block.qs, elem1);
215
- let q2 = get_q4(block.qs, elem2);
216
- let q3 = get_q4(block.qs, elem3);
217
-
218
- let w0 = scale * f32(q0) - min_val;
219
- let w1 = scale * f32(q1) - min_val;
220
- let w2 = scale * f32(q2) - min_val;
221
- let w3 = scale * f32(q3) - min_val;
222
-
223
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
224
- }
225
- }
226
- }
227
- }
228
- } // end if (is_valid)
229
-
230
- // Subgroup reduction - ALL threads must execute (uniform control flow)
231
- let sg_sum = subgroupAdd(partial_sum);
232
-
233
- // Inter-subgroup reduction via shared memory
234
- let subgroup_id = local_id / sg_size;
235
- let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
236
-
237
- if (sg_id == 0u) {
238
- wg_sums[subgroup_id] = sg_sum;
239
- }
240
-
241
- workgroupBarrier();
242
-
243
- // Thread 0 does final reduction and writes result (only if valid column)
244
- if (local_id == 0u && is_valid) {
245
- var final_sum: f32 = 0.0;
246
- for (var i: u32 = 0u; i < num_subgroups; i = i + 1u) {
247
- final_sum = final_sum + wg_sums[i];
248
- }
249
- C[col] = final_sum * u.alpha;
250
- }
251
- }
252
101
 
253
102
  // ============================================================================
254
103
  // Multi-column GEMV for large vocab (LM head)
@@ -305,7 +154,7 @@ fn main_multicol(
305
154
  let k_base = b * QK_K;
306
155
 
307
156
  // Process all 8 sub-blocks
308
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
157
+ for (var sb: u32 = 0u; sb < NUM_SUBBLOCKS; sb = sb + 1u) {
309
158
  let sm = get_scale_min_k4(block.scales, sb);
310
159
  let scale = d * f32(sm.x);
311
160
  let min_val = dmin * f32(sm.y);
@@ -346,7 +195,7 @@ fn main_multicol(
346
195
  let dmin = unpack_f16_hi(block.d_dmin);
347
196
  let k_base = tail_block * QK_K;
348
197
 
349
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
198
+ for (var sb: u32 = 0u; sb < NUM_SUBBLOCKS; sb = sb + 1u) {
350
199
  let sb_base = sb * SUBBLOCK_SIZE;
351
200
  if (sb_base >= tail_size) {
352
201
  break;
@@ -403,290 +252,25 @@ fn main_multicol(
403
252
  }
404
253
  }
405
254
 
406
- // ============================================================================
407
- // Fast-dequant variant: replaces per-element get_q4() calls (~10 ALU each)
408
- // with direct word extraction (~6 ALU per 4 elements).
409
- // Q4_K sub-block layout: even sub-blocks use lower nibbles, odd use upper.
410
- // 4 consecutive bytes (1 u32 word) yield 4 q4 values via shift+mask.
411
- // ============================================================================
412
- var<workgroup> multicol_sums_fd: array<f32, MAX_WORKGROUP_SIZE>;
413
-
414
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
415
- fn main_multicol_fast(
416
- @builtin(local_invocation_id) lid: vec3<u32>,
417
- @builtin(workgroup_id) wg_id: vec3<u32>,
418
- @builtin(subgroup_invocation_id) sg_id: u32,
419
- @builtin(subgroup_size) sg_size: u32
420
- ) {
421
- let local_id = lid.x;
422
-
423
- let col_in_wg = local_id / THREADS_PER_COL_GEMV;
424
- let tid_in_col = local_id % THREADS_PER_COL_GEMV;
425
-
426
- let col = wg_id.x * COLS_PER_WG + col_in_wg;
427
-
428
- let is_valid = col < u.N;
429
-
430
- var partial_sum: f32 = 0.0;
431
-
432
- if (is_valid) {
433
- let num_blocks = u.num_blocks_per_row;
434
- let tail_size = u.K & 255u;
435
- let full_blocks = num_blocks - select(0u, 1u, tail_size > 0u);
436
-
437
- for (var b: u32 = tid_in_col; b < full_blocks; b = b + THREADS_PER_COL_GEMV) {
438
- let block = B_q4k[col * num_blocks + b];
439
- let d = unpack_f16_lo(block.d_dmin);
440
- let dmin = unpack_f16_hi(block.d_dmin);
441
- let k_base = b * QK_K;
442
-
443
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
444
- let sm = get_scale_min_k4(block.scales, sb);
445
- let scale = d * f32(sm.x);
446
- let min_val = dmin * f32(sm.y);
447
- let sb_base = sb * SUBBLOCK_SIZE;
448
-
449
- // Direct word extraction: even sb = lower nibble, odd sb = upper
450
- let chunk = sb >> 1u;
451
- let nibble_shift = (sb & 1u) * 4u;
452
- let word_base = chunk * 8u;
453
-
454
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
455
- let k0 = k_base + sb_base + i;
456
-
457
- let a0 = A[k0];
458
- let a1 = A[k0 + 1u];
459
- let a2 = A[k0 + 2u];
460
- let a3 = A[k0 + 3u];
461
-
462
- let word = block.qs[word_base + (i >> 2u)];
463
- let q0 = (word >> nibble_shift) & 0xFu;
464
- let q1 = (word >> (nibble_shift + 8u)) & 0xFu;
465
- let q2 = (word >> (nibble_shift + 16u)) & 0xFu;
466
- let q3 = (word >> (nibble_shift + 24u)) & 0xFu;
467
-
468
- let w0 = scale * f32(q0) - min_val;
469
- let w1 = scale * f32(q1) - min_val;
470
- let w2 = scale * f32(q2) - min_val;
471
- let w3 = scale * f32(q3) - min_val;
472
-
473
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
474
- }
475
- }
476
- }
477
-
478
- if (tail_size > 0u) {
479
- let tail_block = full_blocks;
480
- if (tail_block % THREADS_PER_COL_GEMV == tid_in_col) {
481
- let block = B_q4k[col * num_blocks + tail_block];
482
- let d = unpack_f16_lo(block.d_dmin);
483
- let dmin = unpack_f16_hi(block.d_dmin);
484
- let k_base = tail_block * QK_K;
485
-
486
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
487
- let sb_base = sb * SUBBLOCK_SIZE;
488
- if (sb_base >= tail_size) {
489
- break;
490
- }
491
- let sm = get_scale_min_k4(block.scales, sb);
492
- let scale = d * f32(sm.x);
493
- let min_val = dmin * f32(sm.y);
494
-
495
- let chunk = sb >> 1u;
496
- let nibble_shift = (sb & 1u) * 4u;
497
- let word_base = chunk * 8u;
498
-
499
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
500
- let k0 = k_base + sb_base + i;
501
-
502
- var a0: f32 = 0.0;
503
- var a1: f32 = 0.0;
504
- var a2: f32 = 0.0;
505
- var a3: f32 = 0.0;
506
- if (k0 < u.K) { a0 = A[k0]; }
507
- if (k0 + 1u < u.K) { a1 = A[k0 + 1u]; }
508
- if (k0 + 2u < u.K) { a2 = A[k0 + 2u]; }
509
- if (k0 + 3u < u.K) { a3 = A[k0 + 3u]; }
510
-
511
- let word = block.qs[word_base + (i >> 2u)];
512
- let q0 = (word >> nibble_shift) & 0xFu;
513
- let q1 = (word >> (nibble_shift + 8u)) & 0xFu;
514
- let q2 = (word >> (nibble_shift + 16u)) & 0xFu;
515
- let q3 = (word >> (nibble_shift + 24u)) & 0xFu;
516
-
517
- let w0 = scale * f32(q0) - min_val;
518
- let w1 = scale * f32(q1) - min_val;
519
- let w2 = scale * f32(q2) - min_val;
520
- let w3 = scale * f32(q3) - min_val;
521
-
522
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
523
- }
524
- }
525
- }
526
- }
527
- }
528
-
529
- multicol_sums_fd[local_id] = partial_sum;
530
- workgroupBarrier();
531
-
532
- if (tid_in_col == 0u && is_valid) {
533
- var final_sum: f32 = 0.0;
534
- let base = col_in_wg * THREADS_PER_COL_GEMV;
535
- for (var i: u32 = 0u; i < THREADS_PER_COL_GEMV; i = i + 1u) {
536
- final_sum = final_sum + multicol_sums_fd[base + i];
537
- }
538
- C[col] = final_sum * u.alpha;
539
- }
540
- }
541
255
 
542
- // ============================================================================
543
- // Shared-A variant: loads the A vector into workgroup memory once,
544
- // eliminating redundant global reads across columns.
545
- // Use when K <= SHARED_A_MAX (compile-time override).
546
- // Total shared memory: SHARED_A_MAX*4 + 256*4 = e.g. 2048*4+1024 = 9216 bytes.
547
- // ============================================================================
548
256
  override SHARED_A_MAX: u32 = 3584u;
549
257
 
550
- var<workgroup> shared_A_buf: array<f32, SHARED_A_MAX>;
551
- var<workgroup> multicol_sums_sa: array<f32, MAX_WORKGROUP_SIZE>;
552
-
553
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
554
- fn main_multicol_shared(
555
- @builtin(local_invocation_id) lid: vec3<u32>,
556
- @builtin(workgroup_id) wg_id: vec3<u32>,
557
- @builtin(subgroup_invocation_id) sg_id: u32,
558
- @builtin(subgroup_size) sg_size: u32
559
- ) {
560
- let local_id = lid.x;
561
-
562
- // Cooperative load: all 256 threads fill shared_A_buf from global A
563
- for (var idx: u32 = local_id; idx < u.K; idx = idx + WORKGROUP_SIZE) {
564
- shared_A_buf[idx] = A[idx];
565
- }
566
- workgroupBarrier();
567
-
568
- // Which column within this workgroup (0..31)
569
- let col_in_wg = local_id / THREADS_PER_COL_GEMV;
570
- // Which thread within the column (0..7)
571
- let tid_in_col = local_id % THREADS_PER_COL_GEMV;
572
-
573
- // Global column index
574
- let col = wg_id.x * COLS_PER_WG + col_in_wg;
575
-
576
- // Track validity
577
- let is_valid = col < u.N;
578
-
579
- var partial_sum: f32 = 0.0;
580
-
581
- if (is_valid) {
582
- let num_blocks = u.num_blocks_per_row;
583
- let tail_size = u.K & 255u;
584
- let full_blocks = num_blocks - select(0u, 1u, tail_size > 0u);
585
-
586
- for (var b: u32 = tid_in_col; b < full_blocks; b = b + THREADS_PER_COL_GEMV) {
587
- let block = B_q4k[col * num_blocks + b];
588
- let d = unpack_f16_lo(block.d_dmin);
589
- let dmin = unpack_f16_hi(block.d_dmin);
590
- let k_base = b * QK_K;
591
-
592
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
593
- let sm = get_scale_min_k4(block.scales, sb);
594
- let scale = d * f32(sm.x);
595
- let min_val = dmin * f32(sm.y);
596
- let sb_base = sb * SUBBLOCK_SIZE;
597
-
598
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
599
- let k0 = k_base + sb_base + i;
600
-
601
- let a0 = shared_A_buf[k0];
602
- let a1 = shared_A_buf[k0 + 1u];
603
- let a2 = shared_A_buf[k0 + 2u];
604
- let a3 = shared_A_buf[k0 + 3u];
605
-
606
- let q0 = get_q4(block.qs, sb_base + i);
607
- let q1 = get_q4(block.qs, sb_base + i + 1u);
608
- let q2 = get_q4(block.qs, sb_base + i + 2u);
609
- let q3 = get_q4(block.qs, sb_base + i + 3u);
610
-
611
- let w0 = scale * f32(q0) - min_val;
612
- let w1 = scale * f32(q1) - min_val;
613
- let w2 = scale * f32(q2) - min_val;
614
- let w3 = scale * f32(q3) - min_val;
615
-
616
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
617
- }
618
- }
619
- }
620
-
621
- if (tail_size > 0u) {
622
- let tail_block = full_blocks;
623
- if (tail_block % THREADS_PER_COL_GEMV == tid_in_col) {
624
- let block = B_q4k[col * num_blocks + tail_block];
625
- let d = unpack_f16_lo(block.d_dmin);
626
- let dmin = unpack_f16_hi(block.d_dmin);
627
- let k_base = tail_block * QK_K;
628
-
629
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
630
- let sb_base = sb * SUBBLOCK_SIZE;
631
- if (sb_base >= tail_size) {
632
- break;
633
- }
634
- let sm = get_scale_min_k4(block.scales, sb);
635
- let scale = d * f32(sm.x);
636
- let min_val = dmin * f32(sm.y);
637
-
638
- for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
639
- let k0 = k_base + sb_base + i;
640
-
641
- var a0: f32 = 0.0;
642
- var a1: f32 = 0.0;
643
- var a2: f32 = 0.0;
644
- var a3: f32 = 0.0;
645
- if (k0 < u.K) { a0 = shared_A_buf[k0]; }
646
- if (k0 + 1u < u.K) { a1 = shared_A_buf[k0 + 1u]; }
647
- if (k0 + 2u < u.K) { a2 = shared_A_buf[k0 + 2u]; }
648
- if (k0 + 3u < u.K) { a3 = shared_A_buf[k0 + 3u]; }
649
-
650
- let q0 = get_q4(block.qs, sb_base + i);
651
- let q1 = get_q4(block.qs, sb_base + i + 1u);
652
- let q2 = get_q4(block.qs, sb_base + i + 2u);
653
- let q3 = get_q4(block.qs, sb_base + i + 3u);
654
-
655
- let w0 = scale * f32(q0) - min_val;
656
- let w1 = scale * f32(q1) - min_val;
657
- let w2 = scale * f32(q2) - min_val;
658
- let w3 = scale * f32(q3) - min_val;
659
-
660
- partial_sum = partial_sum + a0 * w0 + a1 * w1 + a2 * w2 + a3 * w3;
661
- }
662
- }
663
- }
664
- }
665
- }
666
-
667
- // Reduction within each column's 8 threads
668
- multicol_sums_sa[local_id] = partial_sum;
669
- workgroupBarrier();
670
-
671
- if (tid_in_col == 0u && is_valid) {
672
- var final_sum: f32 = 0.0;
673
- let base = col_in_wg * THREADS_PER_COL_GEMV;
674
- for (var i: u32 = 0u; i < THREADS_PER_COL_GEMV; i = i + 1u) {
675
- final_sum = final_sum + multicol_sums_sa[base + i];
676
- }
677
- C[col] = final_sum * u.alpha;
678
- }
679
- }
680
258
 
681
259
  // ============================================================================
682
260
  // Optimised GEMV: shared-A cooperative load + fast nibble extraction.
683
- // Combines the shared A buffer from main_multicol_shared with the direct
684
- // word extraction from main_multicol_fast for maximum M=1 decode throughput.
685
- // Use when K <= SHARED_A_MAX (compile-time override, default 3584).
261
+ // Uses shared A for the hot prefix and falls back to global A reads when
262
+ // K exceeds SHARED_A_MAX, which keeps large FFN down-projections correct.
686
263
  // ============================================================================
687
264
  var<workgroup> gemv_shared_A: array<f32, SHARED_A_MAX>;
688
265
  var<workgroup> gemv_sums: array<f32, MAX_WORKGROUP_SIZE>;
689
266
 
267
+ fn gemv_load_a(k: u32) -> f32 {
268
+ if (k < SHARED_A_MAX) {
269
+ return gemv_shared_A[k];
270
+ }
271
+ return A[k];
272
+ }
273
+
690
274
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
691
275
  fn main_gemv(
692
276
  @builtin(local_invocation_id) lid: vec3<u32>,
@@ -697,7 +281,7 @@ fn main_gemv(
697
281
  let local_id = lid.x;
698
282
 
699
283
  // Cooperative load: all 256 threads fill shared A from global memory once.
700
- for (var idx: u32 = local_id; idx < u.K; idx = idx + WORKGROUP_SIZE) {
284
+ for (var idx: u32 = local_id; idx < min(u.K, SHARED_A_MAX); idx = idx + WORKGROUP_SIZE) {
701
285
  gemv_shared_A[idx] = A[idx];
702
286
  }
703
287
  workgroupBarrier();
@@ -720,7 +304,7 @@ fn main_gemv(
720
304
  let dmin = unpack_f16_hi(block.d_dmin);
721
305
  let k_base = b * QK_K;
722
306
 
723
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
307
+ for (var sb: u32 = 0u; sb < NUM_SUBBLOCKS; sb = sb + 1u) {
724
308
  let sm = get_scale_min_k4(block.scales, sb);
725
309
  let scale = d * f32(sm.x);
726
310
  let min_val = dmin * f32(sm.y);
@@ -735,10 +319,10 @@ fn main_gemv(
735
319
  for (var i: u32 = 0u; i < SUBBLOCK_SIZE; i = i + 4u) {
736
320
  let k0 = k_base + sb_base + i;
737
321
 
738
- let a0 = gemv_shared_A[k0];
739
- let a1 = gemv_shared_A[k0 + 1u];
740
- let a2 = gemv_shared_A[k0 + 2u];
741
- let a3 = gemv_shared_A[k0 + 3u];
322
+ let a0 = gemv_load_a(k0);
323
+ let a1 = gemv_load_a(k0 + 1u);
324
+ let a2 = gemv_load_a(k0 + 2u);
325
+ let a3 = gemv_load_a(k0 + 3u);
742
326
 
743
327
  let word = block.qs[word_base + (i >> 2u)];
744
328
  let q0 = (word >> nibble_shift) & 0xFu;
@@ -764,7 +348,7 @@ fn main_gemv(
764
348
  let dmin = unpack_f16_hi(block.d_dmin);
765
349
  let k_base = tail_block * QK_K;
766
350
 
767
- for (var sb: u32 = 0u; sb < 8u; sb = sb + 1u) {
351
+ for (var sb: u32 = 0u; sb < NUM_SUBBLOCKS; sb = sb + 1u) {
768
352
  let sb_base = sb * SUBBLOCK_SIZE;
769
353
  if (sb_base >= tail_size) {
770
354
  break;
@@ -784,10 +368,10 @@ fn main_gemv(
784
368
  var a1: f32 = 0.0;
785
369
  var a2: f32 = 0.0;
786
370
  var a3: f32 = 0.0;
787
- if (k0 < u.K) { a0 = gemv_shared_A[k0]; }
788
- if (k0 + 1u < u.K) { a1 = gemv_shared_A[k0 + 1u]; }
789
- if (k0 + 2u < u.K) { a2 = gemv_shared_A[k0 + 2u]; }
790
- if (k0 + 3u < u.K) { a3 = gemv_shared_A[k0 + 3u]; }
371
+ if (k0 < u.K) { a0 = gemv_load_a(k0); }
372
+ if (k0 + 1u < u.K) { a1 = gemv_load_a(k0 + 1u); }
373
+ if (k0 + 2u < u.K) { a2 = gemv_load_a(k0 + 2u); }
374
+ if (k0 + 3u < u.K) { a3 = gemv_load_a(k0 + 3u); }
791
375
 
792
376
  let word = block.qs[word_base + (i >> 2u)];
793
377
  let q0 = (word >> nibble_shift) & 0xFu;
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/fused_matmul_q4_batched.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused Q4_K Matmul Kernel - W4A16 (F16 output, batched)
4
4
  //
5
5
  // Batched version for prefill (M > 1) with f16 output.
@@ -6,7 +6,6 @@ enable f16;
6
6
  enable subgroups;
7
7
 
8
8
  const QK_K: u32 = 256u;
9
- const BLOCK_SIZE: u32 = 144u;
10
9
  const SUBBLOCK_SIZE: u32 = 32u;
11
10
 
12
11
  struct Uniforms {