doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,261 @@
1
+ // Register-Tiled Q4_K Matmul for Prefill — F32 activations, F16 output, Q4_K weights
2
+ //
3
+ // Target shape: M >= 16 (prefill), Q4_K weights, F32 activations, F16 output.
4
+ // Matches q4_fused_batched_f16 binding contract (A=f32, C=f16) so it slots
5
+ // into the existing Q4_K matmul selection path when the caller has already
6
+ // resolved to that variant.
7
+ //
8
+ // C[M,N] = A[M,K] * dequant(B_q4k)^T (transpose_b implicit: weights row-major [N, K])
9
+ //
10
+ // Design:
11
+ // - 16 x 16 = 256 threads per workgroup
12
+ // - Each thread computes a 4 x 4 register tile of outputs (16 outputs)
13
+ // - Per workgroup: TILE_M=64 rows x TILE_N=64 cols = 4096 outputs
14
+ // - K-tile: 16 elements per iteration (TILE_K divides QK_K=256)
15
+ // - Shared memory: tileA (64*16 f32) + tileB (64*16 f16) = 6 KB
16
+ // - Q4_K dequantization runs inline when loading tileB, reusing helpers from
17
+ // fused_matmul_q4_batched_f16.wgsl. Weights are read once into shared
18
+ // memory and then reused across the 4 x 4 register tile, amortizing
19
+ // dequant cost 16x vs per-output dequant kernels.
20
+ //
21
+ // Dispatch convention (Q4_K): workgroupsX = ceil(N / colsPerWg=64),
22
+ // workgroupsY = ceil(M / tileM=64). wg_id.x indexes N-column tile;
23
+ // wg_id.y indexes M-row tile.
24
+ //
25
+ // For Gemma 4 E2B prefill (M=64, N=6144, K=1536):
26
+ // workgroups = (96, 1, 1) = 96 WGs per matmul
27
+ // vs fused_matmul_q4_batched_f16 (TILE_M=4, 1 col/WG): 98,304 WGs — 1024x reduction.
28
+
29
+ enable f16;
30
+
31
+ const QK_K: u32 = 256u;
32
+ const SUBBLOCK_SIZE: u32 = 32u;
33
+
34
+ const TILE_M: u32 = 64u;
35
+ const TILE_N: u32 = 64u;
36
+ const TILE_K: u32 = 16u;
37
+
38
+ const THREAD_M: u32 = 4u;
39
+ const THREAD_N: u32 = 4u;
40
+
41
+ const WG_M: u32 = 16u; // 16 threads along M covers 64 rows (4 per thread)
42
+ const WG_N: u32 = 16u; // 16 threads along N covers 64 cols (4 per thread)
43
+
44
+ struct Uniforms {
45
+ M: u32,
46
+ N: u32,
47
+ K: u32,
48
+ alpha: f32,
49
+ num_blocks_per_row: u32,
50
+ _pad0: u32,
51
+ _pad1: u32,
52
+ _pad2: u32,
53
+ }
54
+
55
+ struct Q4KBlock {
56
+ d_dmin: u32,
57
+ scales: array<u32, 3>,
58
+ qs: array<u32, 32>,
59
+ }
60
+
61
+ @group(0) @binding(0) var<uniform> u: Uniforms;
62
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
63
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
64
+ @group(0) @binding(4) var<storage, read_write> C: array<f16>;
65
+
66
+ var<workgroup> tileA: array<f32, 1024>; // TILE_M * TILE_K = 64 * 16
67
+ var<workgroup> tileB: array<f16, 1024>; // TILE_N * TILE_K = 64 * 16
68
+
69
+ fn unpack_f16_lo(packed: u32) -> f32 {
70
+ return unpack2x16float(packed).x;
71
+ }
72
+
73
+ fn unpack_f16_hi(packed: u32) -> f32 {
74
+ return unpack2x16float(packed).y;
75
+ }
76
+
77
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
78
+ let word_idx = byte_idx / 4u;
79
+ let byte_in_word = byte_idx % 4u;
80
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
81
+ }
82
+
83
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
84
+ var sc: u32;
85
+ var mn: u32;
86
+
87
+ if (j < 4u) {
88
+ sc = get_scale_byte(scales, j) & 63u;
89
+ mn = get_scale_byte(scales, j + 4u) & 63u;
90
+ } else {
91
+ let q_j = get_scale_byte(scales, j + 4u);
92
+ let q_lo = get_scale_byte(scales, j - 4u);
93
+ let q_hi = get_scale_byte(scales, j);
94
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
95
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
96
+ }
97
+ return vec2<u32>(sc, mn);
98
+ }
99
+
100
+ fn get_q4(qs: array<u32, 32>, idx: u32) -> u32 {
101
+ let chunk = idx / 64u;
102
+ let pos_in_chunk = idx % 64u;
103
+ let use_upper = pos_in_chunk >= 32u;
104
+ let byte_in_range = select(pos_in_chunk, pos_in_chunk - 32u, use_upper);
105
+ let byte_idx = chunk * 32u + byte_in_range;
106
+
107
+ let word_idx = byte_idx / 4u;
108
+ let byte_in_word = byte_idx % 4u;
109
+ let byte_val = (qs[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
110
+
111
+ if (use_upper) {
112
+ return (byte_val >> 4u) & 0xFu;
113
+ } else {
114
+ return byte_val & 0xFu;
115
+ }
116
+ }
117
+
118
+ @compute @workgroup_size(WG_M, WG_N, 1)
119
+ fn main(
120
+ @builtin(local_invocation_id) lid: vec3<u32>,
121
+ @builtin(workgroup_id) wg_id: vec3<u32>
122
+ ) {
123
+ let tx = lid.x; // 0..15, row-local
124
+ let ty = lid.y; // 0..15, col-local
125
+ let tid = tx * WG_N + ty; // 0..255, linear thread id
126
+
127
+ // Q4K dispatch convention: workgroupsX = ceil(N/colsPerWg), workgroupsY = ceil(M/tileM).
128
+ // wg_id.x indexes the N-column tile; wg_id.y indexes the M-row tile.
129
+ let row_base = wg_id.y * TILE_M + tx * THREAD_M;
130
+ let col_base = wg_id.x * TILE_N + ty * THREAD_N;
131
+
132
+ // 4x4 f32 accumulators (16 per thread)
133
+ var acc00: f32 = 0.0; var acc01: f32 = 0.0; var acc02: f32 = 0.0; var acc03: f32 = 0.0;
134
+ var acc10: f32 = 0.0; var acc11: f32 = 0.0; var acc12: f32 = 0.0; var acc13: f32 = 0.0;
135
+ var acc20: f32 = 0.0; var acc21: f32 = 0.0; var acc22: f32 = 0.0; var acc23: f32 = 0.0;
136
+ var acc30: f32 = 0.0; var acc31: f32 = 0.0; var acc32: f32 = 0.0; var acc33: f32 = 0.0;
137
+
138
+ let num_k_tiles = (u.K + TILE_K - 1u) / TILE_K;
139
+
140
+ for (var t: u32 = 0u; t < num_k_tiles; t = t + 1u) {
141
+ let k_offset = t * TILE_K;
142
+
143
+ // ===== Stage 1: Load A tile (f16) =====
144
+ // 64 rows x 16 K = 1024 elements across 256 threads = 4 per thread.
145
+ {
146
+ let base = tid * 4u;
147
+ for (var i: u32 = 0u; i < 4u; i = i + 1u) {
148
+ let idx = base + i;
149
+ let load_row = idx / TILE_K;
150
+ let load_k = idx % TILE_K;
151
+ let global_row = wg_id.y * TILE_M + load_row;
152
+ let global_k = k_offset + load_k;
153
+ if (global_row < u.M && global_k < u.K) {
154
+ tileA[idx] = A[global_row * u.K + global_k];
155
+ } else {
156
+ tileA[idx] = 0.0;
157
+ }
158
+ }
159
+ }
160
+
161
+ // ===== Stage 2: Load B tile with inline Q4_K dequant =====
162
+ // 64 cols x 16 K = 1024 slots across 256 threads = 4 per thread.
163
+ // Layout: tileB[col_local * TILE_K + k_local]. Thread tid's 4 slots
164
+ // all belong to col_local = tid / 4, with k_local spanning
165
+ // [(tid % 4) * 4, (tid % 4) * 4 + 3]. k_offset is a multiple of
166
+ // TILE_K=16 and TILE_K divides QK_K=256, so all 4 slots fall within
167
+ // the same Q4_K block (no block-crossing per thread).
168
+ {
169
+ let base = tid * 4u;
170
+ let col_local = base / TILE_K;
171
+ let k_local_start = base % TILE_K;
172
+ let global_col = wg_id.x * TILE_N + col_local;
173
+
174
+ if (global_col < u.N) {
175
+ let global_k_base = k_offset + k_local_start;
176
+ let block_idx = global_k_base / QK_K;
177
+ let elem_base = global_k_base - block_idx * QK_K;
178
+
179
+ let block = B_q4k[global_col * u.num_blocks_per_row + block_idx];
180
+ let d = unpack_f16_lo(block.d_dmin);
181
+ let dmin = unpack_f16_hi(block.d_dmin);
182
+
183
+ var last_sb: u32 = 0xFFFFFFFFu;
184
+ var scale: f32 = 0.0;
185
+ var min_val: f32 = 0.0;
186
+
187
+ for (var i: u32 = 0u; i < 4u; i = i + 1u) {
188
+ let elem = elem_base + i;
189
+ let sb = elem / SUBBLOCK_SIZE;
190
+ if (sb != last_sb) {
191
+ let sm = get_scale_min_k4(block.scales, sb);
192
+ scale = d * f32(sm.x);
193
+ min_val = dmin * f32(sm.y);
194
+ last_sb = sb;
195
+ }
196
+ let q = get_q4(block.qs, elem);
197
+ let w = scale * f32(q) - min_val;
198
+ let slot = col_local * TILE_K + k_local_start + i;
199
+ let global_k = k_offset + k_local_start + i;
200
+ if (global_k < u.K) {
201
+ tileB[slot] = f16(w);
202
+ } else {
203
+ tileB[slot] = f16(0.0);
204
+ }
205
+ }
206
+ } else {
207
+ for (var i: u32 = 0u; i < 4u; i = i + 1u) {
208
+ let slot = col_local * TILE_K + k_local_start + i;
209
+ tileB[slot] = f16(0.0);
210
+ }
211
+ }
212
+ }
213
+
214
+ workgroupBarrier();
215
+
216
+ // ===== Stage 3: Register-tile matmul =====
217
+ // tileA: [TILE_M=64, TILE_K=16], indexed as tileA[row_local * TILE_K + k].
218
+ // tileB: [TILE_N=64, TILE_K=16], indexed as tileB[col_local * TILE_K + k].
219
+ // Each thread reads 4 A rows and 4 B cols per k, 4x4 outer product.
220
+ for (var k: u32 = 0u; k < TILE_K; k = k + 1u) {
221
+ let a0 = tileA[(tx * THREAD_M + 0u) * TILE_K + k];
222
+ let a1 = tileA[(tx * THREAD_M + 1u) * TILE_K + k];
223
+ let a2 = tileA[(tx * THREAD_M + 2u) * TILE_K + k];
224
+ let a3 = tileA[(tx * THREAD_M + 3u) * TILE_K + k];
225
+
226
+ let b0 = f32(tileB[(ty * THREAD_N + 0u) * TILE_K + k]);
227
+ let b1 = f32(tileB[(ty * THREAD_N + 1u) * TILE_K + k]);
228
+ let b2 = f32(tileB[(ty * THREAD_N + 2u) * TILE_K + k]);
229
+ let b3 = f32(tileB[(ty * THREAD_N + 3u) * TILE_K + k]);
230
+
231
+ acc00 += a0 * b0; acc01 += a0 * b1; acc02 += a0 * b2; acc03 += a0 * b3;
232
+ acc10 += a1 * b0; acc11 += a1 * b1; acc12 += a1 * b2; acc13 += a1 * b3;
233
+ acc20 += a2 * b0; acc21 += a2 * b1; acc22 += a2 * b2; acc23 += a2 * b3;
234
+ acc30 += a3 * b0; acc31 += a3 * b1; acc32 += a3 * b2; acc33 += a3 * b3;
235
+ }
236
+
237
+ workgroupBarrier();
238
+ }
239
+
240
+ // ===== Stage 4: Write 4x4 output tile =====
241
+ let alpha = u.alpha;
242
+ if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = f16(acc00 * alpha); }
243
+ if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = f16(acc01 * alpha); }
244
+ if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = f16(acc02 * alpha); }
245
+ if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = f16(acc03 * alpha); }
246
+
247
+ if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = f16(acc10 * alpha); }
248
+ if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = f16(acc11 * alpha); }
249
+ if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = f16(acc12 * alpha); }
250
+ if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = f16(acc13 * alpha); }
251
+
252
+ if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = f16(acc20 * alpha); }
253
+ if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = f16(acc21 * alpha); }
254
+ if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = f16(acc22 * alpha); }
255
+ if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = f16(acc23 * alpha); }
256
+
257
+ if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = f16(acc30 * alpha); }
258
+ if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = f16(acc31 * alpha); }
259
+ if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = f16(acc32 * alpha); }
260
+ if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = f16(acc33 * alpha); }
261
+ }
@@ -0,0 +1,164 @@
1
+ // Fused Q4_K Matmul Kernel — WideTile register pattern (F32 A, Q4_K B, F32 C)
2
+ //
3
+ // F32-output twin of fused_matmul_q4_widetile_f16.wgsl. Same register-tiling
4
+ // pattern and unpack4xU8-vectorized dequant — only the output dtype and the
5
+ // final cast differ. Kept as a separate file because the WGSL binding type
6
+ // is part of the kernel ABI; callers that want f32 output select this
7
+ // variant rather than this kernel coercing behind their back.
8
+ //
9
+ // See fused_matmul_q4_widetile_f16.wgsl for the full design rationale.
10
+
11
+ enable f16;
12
+
13
+ const QK_K: u32 = 256u;
14
+ const SUBBLOCK_SIZE: u32 = 32u;
15
+ const NUM_SUBBLOCK_PAIRS: u32 = 4u;
16
+ const U32S_PER_PAIR: u32 = 8u;
17
+
18
+ override TILE_M: u32 = 4u;
19
+ override TILE_N: u32 = 256u;
20
+
21
+ const MAX_TILE_M: u32 = 8u;
22
+
23
+ struct Uniforms {
24
+ M: u32,
25
+ N: u32,
26
+ K: u32,
27
+ alpha: f32,
28
+ num_blocks_per_row: u32,
29
+ _pad0: u32,
30
+ _pad1: u32,
31
+ _pad2: u32,
32
+ }
33
+
34
+ struct Q4KBlock {
35
+ d_dmin: u32,
36
+ scales: array<u32, 3>,
37
+ qs: array<u32, 32>,
38
+ }
39
+
40
+ @group(0) @binding(0) var<uniform> u: Uniforms;
41
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
42
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
43
+ @group(0) @binding(3) var<storage, read_write> C: array<f32>;
44
+
45
+ var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
46
+
47
+ fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
48
+ fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
49
+
50
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
51
+ let word_idx = byte_idx / 4u;
52
+ let byte_in_word = byte_idx % 4u;
53
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
54
+ }
55
+
56
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
57
+ var sc: u32;
58
+ var mn: u32;
59
+ if (j < 4u) {
60
+ sc = get_scale_byte(scales, j) & 63u;
61
+ mn = get_scale_byte(scales, j + 4u) & 63u;
62
+ } else {
63
+ let q_j = get_scale_byte(scales, j + 4u);
64
+ let q_lo = get_scale_byte(scales, j - 4u);
65
+ let q_hi = get_scale_byte(scales, j);
66
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
67
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
68
+ }
69
+ return vec2<u32>(sc, mn);
70
+ }
71
+
72
+ fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
73
+ let a_row_base = row_local * QK_K;
74
+ return vec4<f32>(
75
+ tileA[a_row_base + base_elem + 0u],
76
+ tileA[a_row_base + base_elem + 1u],
77
+ tileA[a_row_base + base_elem + 2u],
78
+ tileA[a_row_base + base_elem + 3u]
79
+ );
80
+ }
81
+
82
+ @compute @workgroup_size(TILE_N, 1, 1)
83
+ fn main(
84
+ @builtin(local_invocation_id) lid: vec3<u32>,
85
+ @builtin(workgroup_id) wg_id: vec3<u32>
86
+ ) {
87
+ if (TILE_M > MAX_TILE_M) { return; }
88
+
89
+ let col_local = lid.x;
90
+ let row_base = wg_id.y * TILE_M;
91
+ let col_base = wg_id.x * TILE_N;
92
+ let col = col_base + col_local;
93
+
94
+ var results: array<f32, MAX_TILE_M>;
95
+ for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
96
+ results[i] = 0.0;
97
+ }
98
+
99
+ let num_blocks = u.num_blocks_per_row;
100
+
101
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
102
+ let total_a_elems = TILE_M * QK_K;
103
+ for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
104
+ let load_row_local = idx / QK_K;
105
+ let load_k = idx % QK_K;
106
+ let global_row = row_base + load_row_local;
107
+ let global_k = b * QK_K + load_k;
108
+ if (global_row < u.M && global_k < u.K) {
109
+ tileA[idx] = A[global_row * u.K + global_k];
110
+ } else {
111
+ tileA[idx] = 0.0;
112
+ }
113
+ }
114
+ workgroupBarrier();
115
+
116
+ if (col < u.N) {
117
+ let block = B_q4k[col * num_blocks + b];
118
+ let d = unpack_f16_lo(block.d_dmin);
119
+ let dmin = unpack_f16_hi(block.d_dmin);
120
+
121
+ for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
122
+ let sm_even = get_scale_min_k4(block.scales, 2u * p);
123
+ let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
124
+ let scale_even = d * f32(sm_even.x);
125
+ let min_even_scalar = dmin * f32(sm_even.y);
126
+ let scale_odd = d * f32(sm_odd.x);
127
+ let min_odd_scalar = dmin * f32(sm_odd.y);
128
+ let min_even = vec4<f32>(min_even_scalar);
129
+ let min_odd = vec4<f32>(min_odd_scalar);
130
+
131
+ let pair_base_even = p * 64u;
132
+ let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
133
+ let qs_base = p * U32S_PER_PAIR;
134
+
135
+ for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
136
+ let packed = block.qs[qs_base + uidx];
137
+ let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
138
+ let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
139
+ let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
140
+ let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
141
+
142
+ let elem_even = pair_base_even + uidx * 4u;
143
+ let elem_odd = pair_base_odd + uidx * 4u;
144
+
145
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
146
+ let a_even = load_a_vec4(m, elem_even);
147
+ let a_odd = load_a_vec4(m, elem_odd);
148
+ results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
149
+ }
150
+ }
151
+ }
152
+ }
153
+ workgroupBarrier();
154
+ }
155
+
156
+ if (col < u.N) {
157
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
158
+ let row = row_base + m;
159
+ if (row < u.M) {
160
+ C[row * u.N + col] = results[m] * u.alpha;
161
+ }
162
+ }
163
+ }
164
+ }
@@ -0,0 +1,198 @@
1
+ // Fused Q4_K Matmul Kernel — WideTile register pattern (F32 A, Q4_K B, F16 C)
2
+ //
3
+ // Adapted from Microsoft ONNX Runtime's matmul_nbits_wide_tile.wgsl.template:
4
+ // https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/contrib_ops/
5
+ // webgpu/quantization/matmul_nbits_wide_tile.wgsl.template (MIT License)
6
+ //
7
+ // Key design: each thread computes ONE output column across TILE_M rows
8
+ // simultaneously, accumulating `TILE_M` f32 results in registers. One
9
+ // workgroup of TILE_N threads handles TILE_M×TILE_N outputs, giving
10
+ // (M/TILE_M) × (N/TILE_N) total workgroups — orders of magnitude fewer
11
+ // than the 1-col-per-WG geometry in fused_matmul_q4_batched_f16.wgsl.
12
+ //
13
+ // Weight format: llama.cpp Q4_K_M (256-element superblocks with 8 subblocks
14
+ // of 32, 6-bit scale/min per subblock, f16 d/dmin per block). This kernel
15
+ // keeps Doppler's existing block-256 format to avoid a reconversion
16
+ // side-quest, but borrows ORT's vectorized dequant: `unpack4xU8` extracts
17
+ // 4 nibbles per instruction instead of the scalar `get_q4` byte-twiddle
18
+ // with divergent `use_upper` branch.
19
+ //
20
+ // Q4_K byte layout (within one block.qs[32] = 128 bytes = 256 4-bit weights):
21
+ // - 4 "chunks" of 64 elements; each chunk = 32 bytes = 8 u32s.
22
+ // - Within a chunk, byte k packs LOWER nibble = elem k in the even subblock,
23
+ // UPPER nibble = elem k in the odd subblock (llama.cpp convention).
24
+ // - So each pair of subblocks (2p, 2p+1) shares u32s [p*8 .. p*8+7]; one
25
+ // u32 provides 4 weights each for the even and odd subblock at the
26
+ // same relative position. Perfect fit for `unpack4xU8`.
27
+ //
28
+ // Dispatch: workgroups = (ceil(N / TILE_N), ceil(M / TILE_M), 1).
29
+ // wg_id.x indexes the N-column tile, wg_id.y indexes the M-row tile.
30
+ //
31
+ // For Gemma 4 E2B prefill gate (M=64, N=6144, K=1536): 48 × 16 = 768 WGs,
32
+ // each producing 4×128 = 512 outputs. Compare to fused_matmul_q4_batched_f16
33
+ // at TILE_M=4, 1 col/WG: 98,304 WGs. 128× reduction.
34
+ //
35
+ // Shared memory: tileA[TILE_M * QK_K] f32 = 4 KB at TILE_M=4.
36
+
37
+ enable f16;
38
+
39
+ const QK_K: u32 = 256u;
40
+ const SUBBLOCK_SIZE: u32 = 32u;
41
+ const NUM_SUBBLOCK_PAIRS: u32 = 4u; // 8 subblocks / 2 per pair
42
+ const U32S_PER_PAIR: u32 = 8u; // 32 bytes / 4 bytes per u32
43
+
44
+ override TILE_M: u32 = 4u;
45
+ override TILE_N: u32 = 256u;
46
+
47
+ const MAX_TILE_M: u32 = 8u;
48
+
49
+ struct Uniforms {
50
+ M: u32,
51
+ N: u32,
52
+ K: u32,
53
+ alpha: f32,
54
+ num_blocks_per_row: u32,
55
+ _pad0: u32,
56
+ _pad1: u32,
57
+ _pad2: u32,
58
+ }
59
+
60
+ struct Q4KBlock {
61
+ d_dmin: u32,
62
+ scales: array<u32, 3>,
63
+ qs: array<u32, 32>,
64
+ }
65
+
66
+ @group(0) @binding(0) var<uniform> u: Uniforms;
67
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
68
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
69
+ @group(0) @binding(4) var<storage, read_write> C: array<f16>;
70
+
71
+ var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
72
+
73
+ fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
74
+ fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
75
+
76
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
77
+ let word_idx = byte_idx / 4u;
78
+ let byte_in_word = byte_idx % 4u;
79
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
80
+ }
81
+
82
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
83
+ var sc: u32;
84
+ var mn: u32;
85
+ if (j < 4u) {
86
+ sc = get_scale_byte(scales, j) & 63u;
87
+ mn = get_scale_byte(scales, j + 4u) & 63u;
88
+ } else {
89
+ let q_j = get_scale_byte(scales, j + 4u);
90
+ let q_lo = get_scale_byte(scales, j - 4u);
91
+ let q_hi = get_scale_byte(scales, j);
92
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
93
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
94
+ }
95
+ return vec2<u32>(sc, mn);
96
+ }
97
+
98
+ fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
99
+ let a_row_base = row_local * QK_K;
100
+ return vec4<f32>(
101
+ tileA[a_row_base + base_elem + 0u],
102
+ tileA[a_row_base + base_elem + 1u],
103
+ tileA[a_row_base + base_elem + 2u],
104
+ tileA[a_row_base + base_elem + 3u]
105
+ );
106
+ }
107
+
108
+ @compute @workgroup_size(TILE_N, 1, 1)
109
+ fn main(
110
+ @builtin(local_invocation_id) lid: vec3<u32>,
111
+ @builtin(workgroup_id) wg_id: vec3<u32>
112
+ ) {
113
+ if (TILE_M > MAX_TILE_M) { return; }
114
+
115
+ let col_local = lid.x;
116
+ let row_base = wg_id.y * TILE_M;
117
+ let col_base = wg_id.x * TILE_N;
118
+ let col = col_base + col_local;
119
+
120
+ var results: array<f32, MAX_TILE_M>;
121
+ for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
122
+ results[i] = 0.0;
123
+ }
124
+
125
+ let num_blocks = u.num_blocks_per_row;
126
+
127
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
128
+ // ===== Cooperative load of tileA: TILE_M rows × QK_K elements =====
129
+ let total_a_elems = TILE_M * QK_K;
130
+ for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
131
+ let load_row_local = idx / QK_K;
132
+ let load_k = idx % QK_K;
133
+ let global_row = row_base + load_row_local;
134
+ let global_k = b * QK_K + load_k;
135
+ if (global_row < u.M && global_k < u.K) {
136
+ tileA[idx] = A[global_row * u.K + global_k];
137
+ } else {
138
+ tileA[idx] = 0.0;
139
+ }
140
+ }
141
+ workgroupBarrier();
142
+
143
+ // ===== Each thread computes its column's dot product with all TILE_M rows =====
144
+ if (col < u.N) {
145
+ let block = B_q4k[col * num_blocks + b];
146
+ let d = unpack_f16_lo(block.d_dmin);
147
+ let dmin = unpack_f16_hi(block.d_dmin);
148
+
149
+ // Iterate 4 subblock-pairs. Each pair covers 64 K-elements (2 sub-
150
+ // blocks of 32) and reads 8 consecutive u32s from block.qs. One
151
+ // `unpack4xU8(packed & 0x0F0F0F0Fu)` extracts 4 weights for the
152
+ // even subblock; `unpack4xU8((packed >> 4) & 0x0F0F0F0Fu)` extracts
153
+ // 4 weights for the odd subblock. Both share the SAME 8 u32 loads.
154
+ for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
155
+ let sm_even = get_scale_min_k4(block.scales, 2u * p);
156
+ let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
157
+ let scale_even = d * f32(sm_even.x);
158
+ let min_even_scalar = dmin * f32(sm_even.y);
159
+ let scale_odd = d * f32(sm_odd.x);
160
+ let min_odd_scalar = dmin * f32(sm_odd.y);
161
+ let min_even = vec4<f32>(min_even_scalar);
162
+ let min_odd = vec4<f32>(min_odd_scalar);
163
+
164
+ let pair_base_even = p * 64u;
165
+ let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
166
+ let qs_base = p * U32S_PER_PAIR;
167
+
168
+ for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
169
+ let packed = block.qs[qs_base + uidx];
170
+ let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
171
+ let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
172
+ let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
173
+ let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
174
+
175
+ let elem_even = pair_base_even + uidx * 4u;
176
+ let elem_odd = pair_base_odd + uidx * 4u;
177
+
178
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
179
+ let a_even = load_a_vec4(m, elem_even);
180
+ let a_odd = load_a_vec4(m, elem_odd);
181
+ results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
182
+ }
183
+ }
184
+ }
185
+ }
186
+ workgroupBarrier();
187
+ }
188
+
189
+ // ===== Write TILE_M outputs for this thread's column =====
190
+ if (col < u.N) {
191
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
192
+ let row = row_base + m;
193
+ if (row < u.M) {
194
+ C[row * u.N + col] = f16(results[m] * u.alpha);
195
+ }
196
+ }
197
+ }
198
+ }