doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,161 @@
1
+ // Fused Q4_K Matmul Kernel - WideTile register pattern (F16 A, Q4_K B, F16 C)
2
+ //
3
+ // W4A16 twin of fused_matmul_q4_widetile_f16.wgsl. The tiling and Q4_K
4
+ // dequantization path are identical, but the activation buffer is f16 so an
5
+ // execution-v1 f16 activation lane can keep prefill projections in f16 without
6
+ // an implicit widening cast.
7
+
8
+ enable f16;
9
+
10
+ const QK_K: u32 = 256u;
11
+ const SUBBLOCK_SIZE: u32 = 32u;
12
+ const NUM_SUBBLOCK_PAIRS: u32 = 4u;
13
+ const U32S_PER_PAIR: u32 = 8u;
14
+
15
+ override TILE_M: u32 = 4u;
16
+ override TILE_N: u32 = 256u;
17
+
18
+ const MAX_TILE_M: u32 = 8u;
19
+
20
+ struct Uniforms {
21
+ M: u32,
22
+ N: u32,
23
+ K: u32,
24
+ alpha: f32,
25
+ num_blocks_per_row: u32,
26
+ _pad0: u32,
27
+ _pad1: u32,
28
+ _pad2: u32,
29
+ }
30
+
31
+ struct Q4KBlock {
32
+ d_dmin: u32,
33
+ scales: array<u32, 3>,
34
+ qs: array<u32, 32>,
35
+ }
36
+
37
+ @group(0) @binding(0) var<uniform> u: Uniforms;
38
+ @group(0) @binding(1) var<storage, read> A: array<f16>;
39
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
40
+ @group(0) @binding(4) var<storage, read_write> C: array<f16>;
41
+
42
+ var<workgroup> tileA: array<f16, MAX_TILE_M * QK_K>;
43
+
44
+ fn unpack_f16_lo(packed: u32) -> f16 { return f16(unpack2x16float(packed).x); }
45
+ fn unpack_f16_hi(packed: u32) -> f16 { return f16(unpack2x16float(packed).y); }
46
+
47
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
48
+ let word_idx = byte_idx / 4u;
49
+ let byte_in_word = byte_idx % 4u;
50
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
51
+ }
52
+
53
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
54
+ var sc: u32;
55
+ var mn: u32;
56
+ if (j < 4u) {
57
+ sc = get_scale_byte(scales, j) & 63u;
58
+ mn = get_scale_byte(scales, j + 4u) & 63u;
59
+ } else {
60
+ let q_j = get_scale_byte(scales, j + 4u);
61
+ let q_lo = get_scale_byte(scales, j - 4u);
62
+ let q_hi = get_scale_byte(scales, j);
63
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
64
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
65
+ }
66
+ return vec2<u32>(sc, mn);
67
+ }
68
+
69
+ fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f16> {
70
+ let a_row_base = row_local * QK_K;
71
+ return vec4<f16>(
72
+ tileA[a_row_base + base_elem + 0u],
73
+ tileA[a_row_base + base_elem + 1u],
74
+ tileA[a_row_base + base_elem + 2u],
75
+ tileA[a_row_base + base_elem + 3u]
76
+ );
77
+ }
78
+
79
+ @compute @workgroup_size(TILE_N, 1, 1)
80
+ fn main(
81
+ @builtin(local_invocation_id) lid: vec3<u32>,
82
+ @builtin(workgroup_id) wg_id: vec3<u32>
83
+ ) {
84
+ if (TILE_M > MAX_TILE_M) { return; }
85
+
86
+ let col_local = lid.x;
87
+ let row_base = wg_id.y * TILE_M;
88
+ let col_base = wg_id.x * TILE_N;
89
+ let col = col_base + col_local;
90
+
91
+ var results: array<f16, MAX_TILE_M>;
92
+ for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
93
+ results[i] = f16(0.0);
94
+ }
95
+
96
+ let num_blocks = u.num_blocks_per_row;
97
+
98
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
99
+ let total_a_elems = TILE_M * QK_K;
100
+ for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
101
+ let load_row_local = idx / QK_K;
102
+ let load_k = idx % QK_K;
103
+ let global_row = row_base + load_row_local;
104
+ let global_k = b * QK_K + load_k;
105
+ if (global_row < u.M && global_k < u.K) {
106
+ tileA[idx] = A[global_row * u.K + global_k];
107
+ } else {
108
+ tileA[idx] = f16(0.0);
109
+ }
110
+ }
111
+ workgroupBarrier();
112
+
113
+ if (col < u.N) {
114
+ let block = B_q4k[col * num_blocks + b];
115
+ let d = unpack_f16_lo(block.d_dmin);
116
+ let dmin = unpack_f16_hi(block.d_dmin);
117
+
118
+ for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
119
+ let sm_even = get_scale_min_k4(block.scales, 2u * p);
120
+ let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
121
+ let scale_even = d * f16(sm_even.x);
122
+ let min_even_scalar = dmin * f16(sm_even.y);
123
+ let scale_odd = d * f16(sm_odd.x);
124
+ let min_odd_scalar = dmin * f16(sm_odd.y);
125
+ let min_even = vec4<f16>(min_even_scalar);
126
+ let min_odd = vec4<f16>(min_odd_scalar);
127
+
128
+ let pair_base_even = p * 64u;
129
+ let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
130
+ let qs_base = p * U32S_PER_PAIR;
131
+
132
+ for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
133
+ let packed = block.qs[qs_base + uidx];
134
+ let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
135
+ let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
136
+ let w_even = scale_even * vec4<f16>(lower_u32) - min_even;
137
+ let w_odd = scale_odd * vec4<f16>(upper_u32) - min_odd;
138
+
139
+ let elem_even = pair_base_even + uidx * 4u;
140
+ let elem_odd = pair_base_odd + uidx * 4u;
141
+
142
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
143
+ let a_even = load_a_vec4(m, elem_even);
144
+ let a_odd = load_a_vec4(m, elem_odd);
145
+ results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
146
+ }
147
+ }
148
+ }
149
+ }
150
+ workgroupBarrier();
151
+ }
152
+
153
+ if (col < u.N) {
154
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
155
+ let row = row_base + m;
156
+ if (row < u.M) {
157
+ C[row * u.N + col] = results[m] * f16(u.alpha);
158
+ }
159
+ }
160
+ }
161
+ }
@@ -0,0 +1,173 @@
1
+ // Fused Q4_K WideTile Matmul + Residual — F32 A, Q4_K B, F32 C
2
+ //
3
+ // Variant of fused_matmul_q4_widetile.wgsl that fuses the downstream
4
+ // residual add into the epilogue:
5
+ //
6
+ // output[m, n] = sum_k(input[m, k] * dequant(B_q4k)[k, n]) + residual[m, n]
7
+ //
8
+ // Targets the o_proj + attn_residual and ffn_down + ffn_residual sites
9
+ // where the subsequent residual add is otherwise a separate dispatch that
10
+ // pays the full ~0.85 ms/dispatch bubble cost on Dawn/Vulkan for ~200 µs
11
+ // of actual elementwise compute.
12
+ //
13
+ // All register-tiling, cooperative tileA load, unpack4xU8 dequant, and
14
+ // online-tile geometry match the base WideTile kernel. Only the final
15
+ // write differs. See fused_matmul_q4_widetile.wgsl for full design notes.
16
+
17
+ enable f16;
18
+
19
+ const QK_K: u32 = 256u;
20
+ const SUBBLOCK_SIZE: u32 = 32u;
21
+ const NUM_SUBBLOCK_PAIRS: u32 = 4u;
22
+ const U32S_PER_PAIR: u32 = 8u;
23
+
24
+ override TILE_M: u32 = 4u;
25
+ override TILE_N: u32 = 256u;
26
+
27
+ const MAX_TILE_M: u32 = 8u;
28
+
29
+ struct Uniforms {
30
+ M: u32,
31
+ N: u32,
32
+ K: u32,
33
+ alpha: f32,
34
+ num_blocks_per_row: u32,
35
+ _pad0: u32,
36
+ _pad1: u32,
37
+ _pad2: u32,
38
+ }
39
+
40
+ struct Q4KBlock {
41
+ d_dmin: u32,
42
+ scales: array<u32, 3>,
43
+ qs: array<u32, 32>,
44
+ }
45
+
46
+ @group(0) @binding(0) var<uniform> u: Uniforms;
47
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
48
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
49
+ @group(0) @binding(3) var<storage, read_write> C: array<f32>;
50
+ @group(0) @binding(4) var<storage, read> residual: array<f32>;
51
+
52
+ var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
53
+
54
+ fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
55
+ fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
56
+
57
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
58
+ let word_idx = byte_idx / 4u;
59
+ let byte_in_word = byte_idx % 4u;
60
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
61
+ }
62
+
63
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
64
+ var sc: u32;
65
+ var mn: u32;
66
+ if (j < 4u) {
67
+ sc = get_scale_byte(scales, j) & 63u;
68
+ mn = get_scale_byte(scales, j + 4u) & 63u;
69
+ } else {
70
+ let q_j = get_scale_byte(scales, j + 4u);
71
+ let q_lo = get_scale_byte(scales, j - 4u);
72
+ let q_hi = get_scale_byte(scales, j);
73
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
74
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
75
+ }
76
+ return vec2<u32>(sc, mn);
77
+ }
78
+
79
+ fn load_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
80
+ let a_row_base = row_local * QK_K;
81
+ return vec4<f32>(
82
+ tileA[a_row_base + base_elem + 0u],
83
+ tileA[a_row_base + base_elem + 1u],
84
+ tileA[a_row_base + base_elem + 2u],
85
+ tileA[a_row_base + base_elem + 3u]
86
+ );
87
+ }
88
+
89
+ @compute @workgroup_size(TILE_N, 1, 1)
90
+ fn main(
91
+ @builtin(local_invocation_id) lid: vec3<u32>,
92
+ @builtin(workgroup_id) wg_id: vec3<u32>
93
+ ) {
94
+ if (TILE_M > MAX_TILE_M) { return; }
95
+
96
+ let col_local = lid.x;
97
+ let row_base = wg_id.y * TILE_M;
98
+ let col_base = wg_id.x * TILE_N;
99
+ let col = col_base + col_local;
100
+
101
+ var results: array<f32, MAX_TILE_M>;
102
+ for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
103
+ results[i] = 0.0;
104
+ }
105
+
106
+ let num_blocks = u.num_blocks_per_row;
107
+
108
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
109
+ let total_a_elems = TILE_M * QK_K;
110
+ for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
111
+ let load_row_local = idx / QK_K;
112
+ let load_k = idx % QK_K;
113
+ let global_row = row_base + load_row_local;
114
+ let global_k = b * QK_K + load_k;
115
+ if (global_row < u.M && global_k < u.K) {
116
+ tileA[idx] = A[global_row * u.K + global_k];
117
+ } else {
118
+ tileA[idx] = 0.0;
119
+ }
120
+ }
121
+ workgroupBarrier();
122
+
123
+ if (col < u.N) {
124
+ let block = B_q4k[col * num_blocks + b];
125
+ let d = unpack_f16_lo(block.d_dmin);
126
+ let dmin = unpack_f16_hi(block.d_dmin);
127
+
128
+ for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
129
+ let sm_even = get_scale_min_k4(block.scales, 2u * p);
130
+ let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
131
+ let scale_even = d * f32(sm_even.x);
132
+ let min_even_scalar = dmin * f32(sm_even.y);
133
+ let scale_odd = d * f32(sm_odd.x);
134
+ let min_odd_scalar = dmin * f32(sm_odd.y);
135
+ let min_even = vec4<f32>(min_even_scalar);
136
+ let min_odd = vec4<f32>(min_odd_scalar);
137
+
138
+ let pair_base_even = p * 64u;
139
+ let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
140
+ let qs_base = p * U32S_PER_PAIR;
141
+
142
+ for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
143
+ let packed = block.qs[qs_base + uidx];
144
+ let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
145
+ let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
146
+ let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
147
+ let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
148
+
149
+ let elem_even = pair_base_even + uidx * 4u;
150
+ let elem_odd = pair_base_odd + uidx * 4u;
151
+
152
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
153
+ let a_even = load_a_vec4(m, elem_even);
154
+ let a_odd = load_a_vec4(m, elem_odd);
155
+ results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
156
+ }
157
+ }
158
+ }
159
+ }
160
+ workgroupBarrier();
161
+ }
162
+
163
+ // ===== Epilogue: matmul_result * alpha + residual =====
164
+ if (col < u.N) {
165
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
166
+ let row = row_base + m;
167
+ if (row < u.M) {
168
+ let offset = row * u.N + col;
169
+ C[offset] = results[m] * u.alpha + residual[offset];
170
+ }
171
+ }
172
+ }
173
+ }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/fused_matmul_rmsnorm.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused GEMV + RMSNorm Kernel (F16 variant)
4
4
  //
5
5
  // For decode (M=1), combines the down projection matmul with RMSNorm in a single kernel:
@@ -0,0 +1,271 @@
1
+ // Fused RMSNorm + Q4_K WideTile Matmul — F32 A, Q4_K B, F32 C
2
+ //
3
+ // Prologue fuses RMSNorm into the WideTile matmul:
4
+ // normed[m, k] = (input[m, k] / rms(input[m, :])) * norm_weight[k]
5
+ // output[m, n] = sum_k(normed[m, k] * dequant(B_q4k)[k, n])
6
+ //
7
+ // Targets the two Gemma-style pre-matmul norms:
8
+ // input_norm → {q,k,v}_proj (each q/k/v call runs norm internally)
9
+ // pre_feedforward_norm → {gate,up}_proj
10
+ //
11
+ // Redundant norm work across q/k/v is negligible (norm: O(K) per row;
12
+ // matmul: O(K*N_per_WG) per row). Each saved standalone rmsnorm dispatch
13
+ // cuts ~0.85 ms of Dawn/Vulkan bubble — the target.
14
+ //
15
+ // Weight offset: when `RMS_NORM_OFFSET = true`, the stored norm weight
16
+ // encodes `(weight - 1.0)` (Gemma family convention). This kernel reads
17
+ // stored_w and applies `1.0 + stored_w` at use-site.
18
+ //
19
+ // Shared memory: tileA[TILE_M][QK_K] f32 = 4 KB per WG block (sliced
20
+ // across K in the existing WideTile loop). The full-row normed tensor
21
+ // is also cached in shared at `row_sq_sum` + `rms` derived per row;
22
+ // we compute sum-of-squares across K chunks in the same outer loop.
23
+ //
24
+ // Two-pass structure: pass 1 accumulates sum_sq across K, computes rms;
25
+ // pass 2 reuses the same tileA buffer to multiply normed*weight and
26
+ // dequant+matmul. Pass 2 reads K in the same block iteration order.
27
+ //
28
+ // Adapted from the iter-8 WideTile design.
29
+
30
+ enable f16;
31
+
32
+ const QK_K: u32 = 256u;
33
+ const SUBBLOCK_SIZE: u32 = 32u;
34
+ const NUM_SUBBLOCK_PAIRS: u32 = 4u;
35
+ const U32S_PER_PAIR: u32 = 8u;
36
+
37
+ override TILE_M: u32 = 4u;
38
+ override TILE_N: u32 = 256u;
39
+ override RMS_NORM_OFFSET: bool = false;
40
+ override WEIGHT_IS_F16: bool = false;
41
+
42
+ const MAX_TILE_M: u32 = 8u;
43
+ const MAX_TILE_N: u32 = 256u;
44
+
45
+ struct Uniforms {
46
+ M: u32,
47
+ N: u32,
48
+ K: u32,
49
+ alpha: f32,
50
+ num_blocks_per_row: u32,
51
+ eps: f32,
52
+ _pad0: u32,
53
+ _pad1: u32,
54
+ }
55
+
56
+ struct Q4KBlock {
57
+ d_dmin: u32,
58
+ scales: array<u32, 3>,
59
+ qs: array<u32, 32>,
60
+ }
61
+
62
+ @group(0) @binding(0) var<uniform> u: Uniforms;
63
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
64
+ @group(0) @binding(2) var<storage, read> B_q4k: array<Q4KBlock>;
65
+ @group(0) @binding(3) var<storage, read_write> C: array<f32>;
66
+ @group(0) @binding(4) var<storage, read> norm_weight: array<u32>;
67
+
68
+ var<workgroup> tileA: array<f32, MAX_TILE_M * QK_K>;
69
+ // Per-row reduction buffer for pass 1: [TILE_M][TILE_N] partial sum-of-squares.
70
+ // Each thread writes its per-row partial, then workgroup-parallel tree reduction
71
+ // collapses to shared_sum_sq[m][0]. Derived row_rms_recip is cached for pass 2.
72
+ var<workgroup> shared_sum_sq: array<array<f32, MAX_TILE_N>, MAX_TILE_M>;
73
+ var<workgroup> row_rms_recip: array<f32, MAX_TILE_M>;
74
+
75
+ fn unpack_f16_lo(packed: u32) -> f32 { return unpack2x16float(packed).x; }
76
+ fn unpack_f16_hi(packed: u32) -> f32 { return unpack2x16float(packed).y; }
77
+
78
+ // norm_weight binding is declared as array<u32> so it can carry either packed
79
+ // f32 values (bitcast) or packed f16 pairs (Gemma-family hidden weights).
80
+ // Mirrors the load_weight pattern in rmsnorm.wgsl.
81
+ fn load_norm_weight(idx: u32) -> f32 {
82
+ if (WEIGHT_IS_F16) {
83
+ let packed = norm_weight[idx >> 1u];
84
+ let pair = unpack2x16float(packed);
85
+ return select(pair.x, pair.y, (idx & 1u) == 1u);
86
+ }
87
+ return bitcast<f32>(norm_weight[idx]);
88
+ }
89
+
90
+ fn get_scale_byte(scales: array<u32, 3>, byte_idx: u32) -> u32 {
91
+ let word_idx = byte_idx / 4u;
92
+ let byte_in_word = byte_idx % 4u;
93
+ return (scales[word_idx] >> (byte_in_word * 8u)) & 0xFFu;
94
+ }
95
+
96
+ fn get_scale_min_k4(scales: array<u32, 3>, j: u32) -> vec2<u32> {
97
+ var sc: u32;
98
+ var mn: u32;
99
+ if (j < 4u) {
100
+ sc = get_scale_byte(scales, j) & 63u;
101
+ mn = get_scale_byte(scales, j + 4u) & 63u;
102
+ } else {
103
+ let q_j = get_scale_byte(scales, j + 4u);
104
+ let q_lo = get_scale_byte(scales, j - 4u);
105
+ let q_hi = get_scale_byte(scales, j);
106
+ sc = (q_j & 0xFu) | ((q_lo >> 6u) << 4u);
107
+ mn = (q_j >> 4u) | ((q_hi >> 6u) << 4u);
108
+ }
109
+ return vec2<u32>(sc, mn);
110
+ }
111
+
112
+ fn load_normed_a_vec4(row_local: u32, base_elem: u32) -> vec4<f32> {
113
+ // tileA at this point holds normed values (pass 2). The rms scaling
114
+ // and norm_weight multiplication have already been applied when tileA
115
+ // was populated for this block. See $main pass-2 section.
116
+ let a_row_base = row_local * QK_K;
117
+ return vec4<f32>(
118
+ tileA[a_row_base + base_elem + 0u],
119
+ tileA[a_row_base + base_elem + 1u],
120
+ tileA[a_row_base + base_elem + 2u],
121
+ tileA[a_row_base + base_elem + 3u]
122
+ );
123
+ }
124
+
125
+ @compute @workgroup_size(TILE_N, 1, 1)
126
+ fn main(
127
+ @builtin(local_invocation_id) lid: vec3<u32>,
128
+ @builtin(workgroup_id) wg_id: vec3<u32>
129
+ ) {
130
+ if (TILE_M > MAX_TILE_M) { return; }
131
+
132
+ let col_local = lid.x;
133
+ let row_base = wg_id.y * TILE_M;
134
+ let col_base = wg_id.x * TILE_N;
135
+ let col = col_base + col_local;
136
+
137
+ var results: array<f32, MAX_TILE_M>;
138
+ for (var i: u32 = 0u; i < TILE_M; i = i + 1u) {
139
+ results[i] = 0.0;
140
+ }
141
+
142
+ let num_blocks = u.num_blocks_per_row;
143
+
144
+ // ===== PASS 1: workgroup-parallel sum-of-squares per row across all K =====
145
+ // Each of TILE_N threads accumulates a per-thread partial across blocks,
146
+ // then a tree reduction over TILE_N lanes finalises the sum per row. The
147
+ // prior serialised 4-thread reduction was the iter-21 perf wall; this
148
+ // restores full workgroup parallelism (log2(TILE_N)=8 reduction steps).
149
+ var local_sum_sq: array<f32, MAX_TILE_M>;
150
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
151
+ local_sum_sq[m] = 0.0;
152
+ }
153
+
154
+ // Streaming accumulation: thread col_local handles k_in_block == col_local
155
+ // across every block. QK_K (256) == TILE_N (256), so the mapping is 1:1
156
+ // and every lane picks up a K-value per block without serialisation.
157
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
158
+ let global_k = b * QK_K + col_local;
159
+ if (global_k < u.K) {
160
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
161
+ let global_row = row_base + m;
162
+ if (global_row < u.M) {
163
+ let v = A[global_row * u.K + global_k];
164
+ local_sum_sq[m] = local_sum_sq[m] + v * v;
165
+ }
166
+ }
167
+ }
168
+ }
169
+
170
+ // Publish partials into shared memory for tree reduction.
171
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
172
+ shared_sum_sq[m][col_local] = local_sum_sq[m];
173
+ }
174
+ workgroupBarrier();
175
+
176
+ // Tree reduction over TILE_N lanes. All TILE_M rows are reduced in
177
+ // parallel at each stride — 8 barriers total (log2(256)).
178
+ for (var stride: u32 = TILE_N / 2u; stride > 0u; stride = stride >> 1u) {
179
+ if (col_local < stride) {
180
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
181
+ shared_sum_sq[m][col_local] =
182
+ shared_sum_sq[m][col_local]
183
+ + shared_sum_sq[m][col_local + stride];
184
+ }
185
+ }
186
+ workgroupBarrier();
187
+ }
188
+
189
+ // Derive 1/rms per row. One thread per row reads the reduced total.
190
+ if (col_local < TILE_M) {
191
+ let mean_sq = shared_sum_sq[col_local][0] / f32(u.K);
192
+ row_rms_recip[col_local] = 1.0 / sqrt(mean_sq + u.eps);
193
+ }
194
+ workgroupBarrier();
195
+
196
+ // ===== PASS 2: reload A, apply rmsnorm + weight, matmul =====
197
+ for (var b: u32 = 0u; b < num_blocks; b = b + 1u) {
198
+ // Cooperative load of tileA raw (re-read from global — cheap vs
199
+ // matmul work), apply norm in place, then matmul. Applying norm
200
+ // in the same cooperative load avoids an extra barrier.
201
+ let total_a_elems = TILE_M * QK_K;
202
+ for (var idx: u32 = col_local; idx < total_a_elems; idx = idx + TILE_N) {
203
+ let load_row_local = idx / QK_K;
204
+ let load_k = idx % QK_K;
205
+ let global_row = row_base + load_row_local;
206
+ let global_k = b * QK_K + load_k;
207
+ var raw: f32 = 0.0;
208
+ if (global_row < u.M && global_k < u.K) {
209
+ raw = A[global_row * u.K + global_k];
210
+ }
211
+ // rmsnorm: x * (1/rms)
212
+ var normed = raw * row_rms_recip[load_row_local];
213
+ // weight: multiply by norm_weight (with Gemma (1+w) offset).
214
+ if (global_k < u.K) {
215
+ let stored_w = load_norm_weight(global_k);
216
+ let effective_w = select(stored_w, 1.0 + stored_w, RMS_NORM_OFFSET);
217
+ normed = normed * effective_w;
218
+ }
219
+ tileA[idx] = normed;
220
+ }
221
+ workgroupBarrier();
222
+
223
+ if (col < u.N) {
224
+ let block = B_q4k[col * num_blocks + b];
225
+ let d = unpack_f16_lo(block.d_dmin);
226
+ let dmin = unpack_f16_hi(block.d_dmin);
227
+
228
+ for (var p: u32 = 0u; p < NUM_SUBBLOCK_PAIRS; p = p + 1u) {
229
+ let sm_even = get_scale_min_k4(block.scales, 2u * p);
230
+ let sm_odd = get_scale_min_k4(block.scales, 2u * p + 1u);
231
+ let scale_even = d * f32(sm_even.x);
232
+ let min_even_scalar = dmin * f32(sm_even.y);
233
+ let scale_odd = d * f32(sm_odd.x);
234
+ let min_odd_scalar = dmin * f32(sm_odd.y);
235
+ let min_even = vec4<f32>(min_even_scalar);
236
+ let min_odd = vec4<f32>(min_odd_scalar);
237
+
238
+ let pair_base_even = p * 64u;
239
+ let pair_base_odd = p * 64u + SUBBLOCK_SIZE;
240
+ let qs_base = p * U32S_PER_PAIR;
241
+
242
+ for (var uidx: u32 = 0u; uidx < U32S_PER_PAIR; uidx = uidx + 1u) {
243
+ let packed = block.qs[qs_base + uidx];
244
+ let lower_u32 = unpack4xU8(packed & 0x0F0F0F0Fu);
245
+ let upper_u32 = unpack4xU8((packed >> 4u) & 0x0F0F0F0Fu);
246
+ let w_even = scale_even * vec4<f32>(lower_u32) - min_even;
247
+ let w_odd = scale_odd * vec4<f32>(upper_u32) - min_odd;
248
+
249
+ let elem_even = pair_base_even + uidx * 4u;
250
+ let elem_odd = pair_base_odd + uidx * 4u;
251
+
252
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
253
+ let a_even = load_normed_a_vec4(m, elem_even);
254
+ let a_odd = load_normed_a_vec4(m, elem_odd);
255
+ results[m] = results[m] + dot(a_even, w_even) + dot(a_odd, w_odd);
256
+ }
257
+ }
258
+ }
259
+ }
260
+ workgroupBarrier();
261
+ }
262
+
263
+ if (col < u.N) {
264
+ for (var m: u32 = 0u; m < TILE_M; m = m + 1u) {
265
+ let row = row_base + m;
266
+ if (row < u.M) {
267
+ C[row * u.N + col] = results[m] * u.alpha;
268
+ }
269
+ }
270
+ }
271
+ }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/fused_swiglu.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // fused_swiglu_f16.wgsl
4
4
 
5
5
  /**
@@ -35,13 +35,6 @@ export interface GatherOptions extends OutputBufferOptions {
35
35
  indirectOffset?: number;
36
36
  }
37
37
 
38
- /** Context for gather variant selection rules. */
39
- export interface GatherSelectionContext {
40
- useF16Input: boolean;
41
- useF16Output: boolean;
42
- useVec4: boolean;
43
- }
44
-
45
38
  /**
46
39
  * Run gather/embedding lookup
47
40
  */
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/gather.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // gather_f16.wgsl
4
4
 
5
5
  /**