doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -29,6 +29,7 @@ import {
29
29
  selectMatmulKernel,
30
30
  } from './matmul-selection.js';
31
31
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
32
+ import { getRuntimeConfig } from '../../config/runtime.js';
32
33
  import {
33
34
  MatmulKernel,
34
35
  calculateMatmulDispatch,
@@ -36,6 +37,7 @@ import {
36
37
  createMatmulBindGroupLayout,
37
38
  getMatmulPipeline,
38
39
  } from './matmul-dispatch.js';
40
+ import { __dbgRecord } from './utils.js';
39
41
 
40
42
  export { isFusedQ4KDisabled, selectMatmulKernel };
41
43
  export { createMatmulBindGroupLayout };
@@ -96,12 +98,22 @@ function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel,
96
98
  );
97
99
  }
98
100
 
99
- function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes) {
101
+ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null) {
100
102
  const isQ4KF16 = variant === 'q4_fused_multicol_f16'
101
103
  || variant === 'q4_fused_f16a'
102
104
  || variant === 'q4_fused_batched_f16'
103
105
  || variant === 'q4_fused_multicol_f16a'
104
- || variant === 'q4_fused_batched_f16a';
106
+ || variant === 'q4_fused_multicol_f16a_f32acc'
107
+ || variant === 'q4_fused_batched_f16a'
108
+ || variant === 'q4_fused_batched_f16acc_f16a'
109
+ || variant === 'q4_fused_prefill_tiled_f16'
110
+ || variant === 'q4_fused_widetile_f16'
111
+ || variant === 'q4_fused_widetile_f16a';
112
+ // 5-entry WideTile epilogue/prologue variants: output at binding 3 + one
113
+ // extra read-only buffer at binding 4 (residual for _residual, norm weight
114
+ // for _rmsnorm). Distinct from isQ4KF16 (which puts output at binding 4).
115
+ const isWideTileResidual = variant === 'q4_fused_widetile_residual';
116
+ const isWideTileRmsnorm = variant === 'q4_fused_rmsnorm_widetile';
105
117
 
106
118
  assertBindGroupBuffer('matmul', variant, 0, 'uniforms', uniformBuffer);
107
119
  assertBindGroupBuffer('matmul', variant, 1, 'input', matmulInput?.buffer, [
@@ -110,6 +122,18 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
110
122
  ]);
111
123
  assertBindGroupBuffer('matmul', variant, 2, 'weights', bBuffer);
112
124
  assertBindGroupBuffer('matmul', variant, isQ4KF16 ? 4 : 3, 'output', outputBuffer);
125
+ if (isWideTileResidual) {
126
+ if (!residualBuffer) {
127
+ throw new Error(`[Matmul] variant "${variant}" requires a residual buffer but none was provided.`);
128
+ }
129
+ assertBindGroupBuffer('matmul', variant, 4, 'residual', residualBuffer);
130
+ }
131
+ if (isWideTileRmsnorm) {
132
+ if (!normWeightBuffer) {
133
+ throw new Error(`[Matmul] variant "${variant}" requires a norm weight buffer but none was provided.`);
134
+ }
135
+ assertBindGroupBuffer('matmul', variant, 4, 'norm_weight', normWeightBuffer);
136
+ }
113
137
 
114
138
  const entries = [
115
139
  { binding: 0, resource: { buffer: uniformBuffer } },
@@ -127,6 +151,18 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
127
151
  binding: 3,
128
152
  resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
129
153
  });
154
+ if (isWideTileResidual) {
155
+ entries.push({
156
+ binding: 4,
157
+ resource: { buffer: residualBuffer },
158
+ });
159
+ }
160
+ if (isWideTileRmsnorm) {
161
+ entries.push({
162
+ binding: 4,
163
+ resource: { buffer: normWeightBuffer },
164
+ });
165
+ }
130
166
  }
131
167
 
132
168
  return entries;
@@ -214,7 +250,23 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
214
250
 
215
251
  validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
216
252
 
217
- const { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
253
+ const runtimeSession = getRuntimeConfig().inference?.session;
254
+ const effectiveOptions = (
255
+ options.useTiledQ4KPrefill == null
256
+ || options.useWideTileQ4KPrefill == null
257
+ || options.useWideTileResidualFusion == null
258
+ || options.useFusedRmsnormWideTile == null
259
+ )
260
+ ? {
261
+ ...options,
262
+ useTiledQ4KPrefill: options.useTiledQ4KPrefill ?? (runtimeSession?.useTiledQ4KPrefill === true),
263
+ useWideTileQ4KPrefill: options.useWideTileQ4KPrefill ?? (runtimeSession?.useWideTileQ4KPrefill === true),
264
+ useWideTileResidualFusion: options.useWideTileResidualFusion ?? (runtimeSession?.useWideTileResidualFusion === true),
265
+ useFusedRmsnormWideTile: options.useFusedRmsnormWideTile ?? (runtimeSession?.useFusedRmsnormWideTile === true),
266
+ }
267
+ : options;
268
+
269
+ let { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
218
270
  mode,
219
271
  M,
220
272
  N,
@@ -223,10 +275,35 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
223
275
  bDtype,
224
276
  transposeB,
225
277
  requestedOutputDtype,
226
- options
278
+ effectiveOptions
227
279
  );
228
280
 
229
- const constants = resolveMatmulConstants(options, phase);
281
+ if (
282
+ runtimeSession?.useF32AccumF16ioMatmul === true
283
+ && useQ4KFused
284
+ && variant === 'q4_fused_multicol_f16a'
285
+ ) {
286
+ variant = 'q4_fused_multicol_f16a_f32acc';
287
+ }
288
+
289
+ let constants = resolveMatmulConstants(options, phase);
290
+ if (variant === 'f32' && constants && options.constants == null) {
291
+ constants = null;
292
+ }
293
+ // For the rmsnorm-fused WideTile variant, forward the caller's
294
+ // rmsNormOffset flag as a pipeline override constant. Gemma-family norm
295
+ // weights encode `(w - 1.0)`; other models encode `w`.
296
+ // Also forward WEIGHT_IS_F16 based on the norm weight buffer dtype so the
297
+ // kernel correctly unpacks f16-packed weights (Gemma hidden weights) vs
298
+ // f32 weights.
299
+ if (variant === 'q4_fused_rmsnorm_widetile' && options.rmsNormOffset != null) {
300
+ const normWeightDtype = getWeightDtype(options.normWeight) ?? 'f32';
301
+ constants = {
302
+ ...(constants ?? {}),
303
+ RMS_NORM_OFFSET: options.rmsNormOffset === true,
304
+ WEIGHT_IS_F16: normWeightDtype === 'f16',
305
+ };
306
+ }
230
307
 
231
308
  let matmulInput = A;
232
309
  let matmulADtype = aDtype;
@@ -325,9 +402,12 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
325
402
  );
326
403
  }
327
404
 
405
+ const __dbg = (typeof process !== "undefined" ? process : { env: {} }).env.DOPPLER_DBG_RECORD === '1';
406
+ const __t0 = __dbg ? performance.now() : 0;
328
407
  const config = getMatmulConfig(variant, constants);
329
408
  const kernel = new MatmulKernel(device);
330
409
  const pipeline = await getMatmulPipeline(variant, constants);
410
+ const __tPipeline = __dbg ? performance.now() : 0;
331
411
 
332
412
  const { output: C, outputSize, cBindingSize, actualOutputDtype } = resolveMatmulOutput(
333
413
  variant,
@@ -358,6 +438,9 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
358
438
  let uniformBuffer = null;
359
439
  let completed = false;
360
440
  try {
441
+ const uniformExtras = variant === 'q4_fused_rmsnorm_widetile' && Number.isFinite(options.rmsNormEps)
442
+ ? { eps: options.rmsNormEps }
443
+ : null;
361
444
  uniformBuffer = createMatmulUniformBuffer(
362
445
  'matmul_uniforms',
363
446
  M,
@@ -368,9 +451,12 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
368
451
  transposeB,
369
452
  dispatchPlan.uniformWorkgroupsX,
370
453
  recorder || null,
371
- device
454
+ device,
455
+ uniformExtras
372
456
  );
373
457
 
458
+ const residualBuffer = options.residualTensor?.buffer ?? null;
459
+ const normWeightBuffer = options.normWeight?.buffer ?? options.normWeight ?? null;
374
460
  const entries = createMatmulBindGroupEntries(
375
461
  variant,
376
462
  uniformBuffer,
@@ -382,20 +468,28 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
382
468
  aBindingSize: bindingSizes.aBindingSize,
383
469
  bBindingSize: bindingSizes.bBindingSize,
384
470
  cBindingSize,
385
- }
471
+ },
472
+ residualBuffer,
473
+ normWeightBuffer
386
474
  );
387
475
 
476
+ const __tBgStart = __dbg ? performance.now() : 0;
388
477
  const bindGroup = device.createBindGroup({
389
478
  label: 'matmul_bind_group',
390
479
  layout: pipeline.getBindGroupLayout(0),
391
480
  entries,
392
481
  });
482
+ const __tBg = __dbg ? performance.now() : 0;
393
483
 
394
484
  if (isRecord) {
395
485
  kernel.record(recorder, pipeline, bindGroup, dispatchPlan.workgroups, buildProfileLabel(options));
396
486
  } else {
397
487
  kernel.dispatch(pipeline, bindGroup, dispatchPlan.workgroups);
398
488
  }
489
+ if (__dbg) {
490
+ const __tEnd = performance.now();
491
+ __dbgRecord('matmul', variant, __tPipeline - __t0, __tBgStart - __tPipeline, __tBg - __tBgStart, __tEnd - __tBg);
492
+ }
399
493
  completed = true;
400
494
  return createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
401
495
  } finally {
@@ -8,7 +8,7 @@
8
8
  // C[M,N] = A[M,K] * B^T[N,K] (transpose_b=1, SafeTensors/RDRR row-major layout)
9
9
  //
10
10
  // Design:
11
- // - f16 storage, f32 accumulators for numerical stability
11
+ // - f16 storage and f16 accumulators for the all-f16 execution lane
12
12
  // - coalesced scalar loads into shared tiles
13
13
  // - Shared memory tiles: 64 x TILE_K for A and B
14
14
  // - B is always assumed row-major (transpose_b=1) for RDRR models
@@ -64,11 +64,10 @@ fn main(
64
64
  let row_base = wg_id.x * TILE_M + tx * THREAD_M;
65
65
  let col_base = wg_id.y * TILE_N + ty * THREAD_N;
66
66
 
67
- // f32 accumulators: 4x4 = 16 per thread
68
- var acc00: f32 = 0.0; var acc01: f32 = 0.0; var acc02: f32 = 0.0; var acc03: f32 = 0.0;
69
- var acc10: f32 = 0.0; var acc11: f32 = 0.0; var acc12: f32 = 0.0; var acc13: f32 = 0.0;
70
- var acc20: f32 = 0.0; var acc21: f32 = 0.0; var acc22: f32 = 0.0; var acc23: f32 = 0.0;
71
- var acc30: f32 = 0.0; var acc31: f32 = 0.0; var acc32: f32 = 0.0; var acc33: f32 = 0.0;
67
+ var acc00: f16 = f16(0.0); var acc01: f16 = f16(0.0); var acc02: f16 = f16(0.0); var acc03: f16 = f16(0.0);
68
+ var acc10: f16 = f16(0.0); var acc11: f16 = f16(0.0); var acc12: f16 = f16(0.0); var acc13: f16 = f16(0.0);
69
+ var acc20: f16 = f16(0.0); var acc21: f16 = f16(0.0); var acc22: f16 = f16(0.0); var acc23: f16 = f16(0.0);
70
+ var acc30: f16 = f16(0.0); var acc31: f16 = f16(0.0); var acc32: f16 = f16(0.0); var acc33: f16 = f16(0.0);
72
71
 
73
72
  let num_tiles = (u.K + TILE_K - 1u) / TILE_K;
74
73
 
@@ -120,16 +119,16 @@ fn main(
120
119
  // B tile: tileB[n_local * TILE_K + k] for cols ty*4..ty*4+3
121
120
  for (var k: u32 = 0u; k < TILE_K; k = k + 1u) {
122
121
  // Load 4 A values for this thread's rows
123
- let a0 = f32(tileA[(tx * THREAD_M + 0u) * TILE_K + k]);
124
- let a1 = f32(tileA[(tx * THREAD_M + 1u) * TILE_K + k]);
125
- let a2 = f32(tileA[(tx * THREAD_M + 2u) * TILE_K + k]);
126
- let a3 = f32(tileA[(tx * THREAD_M + 3u) * TILE_K + k]);
122
+ let a0 = tileA[(tx * THREAD_M + 0u) * TILE_K + k];
123
+ let a1 = tileA[(tx * THREAD_M + 1u) * TILE_K + k];
124
+ let a2 = tileA[(tx * THREAD_M + 2u) * TILE_K + k];
125
+ let a3 = tileA[(tx * THREAD_M + 3u) * TILE_K + k];
127
126
 
128
127
  // Load 4 B values for this thread's columns
129
- let b0 = f32(tileB[(ty * THREAD_N + 0u) * TILE_K + k]);
130
- let b1 = f32(tileB[(ty * THREAD_N + 1u) * TILE_K + k]);
131
- let b2 = f32(tileB[(ty * THREAD_N + 2u) * TILE_K + k]);
132
- let b3 = f32(tileB[(ty * THREAD_N + 3u) * TILE_K + k]);
128
+ let b0 = tileB[(ty * THREAD_N + 0u) * TILE_K + k];
129
+ let b1 = tileB[(ty * THREAD_N + 1u) * TILE_K + k];
130
+ let b2 = tileB[(ty * THREAD_N + 2u) * TILE_K + k];
131
+ let b3 = tileB[(ty * THREAD_N + 3u) * TILE_K + k];
133
132
 
134
133
  // Outer product: 4x4 accumulation
135
134
  acc00 += a0 * b0; acc01 += a0 * b1; acc02 += a0 * b2; acc03 += a0 * b3;
@@ -142,24 +141,24 @@ fn main(
142
141
  }
143
142
 
144
143
  // Write 4x4 output tile
145
- let alpha = u.alpha;
146
- if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = f16(acc00 * alpha); }
147
- if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = f16(acc01 * alpha); }
148
- if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = f16(acc02 * alpha); }
149
- if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = f16(acc03 * alpha); }
150
-
151
- if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = f16(acc10 * alpha); }
152
- if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = f16(acc11 * alpha); }
153
- if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = f16(acc12 * alpha); }
154
- if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = f16(acc13 * alpha); }
155
-
156
- if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = f16(acc20 * alpha); }
157
- if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = f16(acc21 * alpha); }
158
- if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = f16(acc22 * alpha); }
159
- if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = f16(acc23 * alpha); }
160
-
161
- if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = f16(acc30 * alpha); }
162
- if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = f16(acc31 * alpha); }
163
- if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = f16(acc32 * alpha); }
164
- if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = f16(acc33 * alpha); }
144
+ let alpha = f16(u.alpha);
145
+ if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = acc00 * alpha; }
146
+ if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = acc01 * alpha; }
147
+ if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = acc02 * alpha; }
148
+ if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = acc03 * alpha; }
149
+
150
+ if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = acc10 * alpha; }
151
+ if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = acc11 * alpha; }
152
+ if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = acc12 * alpha; }
153
+ if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = acc13 * alpha; }
154
+
155
+ if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = acc20 * alpha; }
156
+ if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = acc21 * alpha; }
157
+ if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = acc22 * alpha; }
158
+ if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = acc23 * alpha; }
159
+
160
+ if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = acc30 * alpha; }
161
+ if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = acc31 * alpha; }
162
+ if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = acc32 * alpha; }
163
+ if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = acc33 * alpha; }
165
164
  }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/matmul_gemv_residual.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused GEMV + Residual Kernel - Single dispatch for output projection + residual add
4
4
  // For M=1 decode: C[N] = A[K] * B^T[K,N] + residual[N]
5
5
  //
@@ -152,7 +152,6 @@ fn main(
152
152
  // Tune-time overrides can retile this without changing entry points.
153
153
  override MULTICOL_COLS_PER_WG: u32 = 32u;
154
154
  override MULTICOL_THREADS_PER_COL: u32 = 8u;
155
- const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
156
155
 
157
156
  // Shared memory for reduction (one slot per thread)
158
157
  var<workgroup> multicol_wg_sums: array<f32, MAX_WORKGROUP_SIZE>;
@@ -8,7 +8,7 @@
8
8
  // 4. Loop unrolling for better ILP
9
9
  //
10
10
  // A is f16 (activations), B is f16 (weights), C is f16.
11
- // Accumulation is performed in f32 for stability.
11
+ // The Gemma 4 31B all-f16 lane uses f16 accumulation on main_multicol.
12
12
  // transpose_b=0: B is [K, N] (GGUF/column-major), access B[k * N + col]
13
13
  // transpose_b=1: B is [N, K] (SafeTensors/row-major), access B[col * K + k]
14
14
  //
@@ -20,15 +20,15 @@ enable subgroups;
20
20
 
21
21
  override WORKGROUP_SIZE: u32 = 256u;
22
22
  const MAX_WORKGROUP_SIZE: u32 = 256u;
23
- const COLS_PER_WG: u32 = 4u; // Each workgroup computes 4 output columns
24
- const THREADS_PER_COL: u32 = 64u; // 256 / 4 = 64 threads per column
25
- const MAX_SUBGROUPS_PER_COL: u32 = 16u; // Support sg_size >= 4 (64/4 = 16)
23
+ override COLS_PER_WG: u32 = 4u; // Each workgroup computes 4 output columns
24
+ override THREADS_PER_COL: u32 = 64u; // 256 / 4 = 64 threads per column
25
+ override MAX_SUBGROUPS_PER_COL: u32 = 16u; // Support sg_size >= 4 (64/4 = 16)
26
26
 
27
27
  // Alternative layout: more columns per workgroup, fewer threads per column.
28
28
  // This reduces workgroup count and improves occupancy for mid-sized K.
29
- const COLS_PER_WG_COLS8: u32 = 8u; // 256 / 8 = 32 threads per column
30
- const THREADS_PER_COL_COLS8: u32 = 32u;
31
- const MAX_SUBGROUPS_PER_COL_COLS8: u32 = 8u; // Support sg_size >= 4 (32/4 = 8)
29
+ override COLS_PER_WG_COLS8: u32 = 8u; // 256 / 8 = 32 threads per column
30
+ override THREADS_PER_COL_COLS8: u32 = 32u;
31
+ override MAX_SUBGROUPS_PER_COL_COLS8: u32 = 8u; // Support sg_size >= 4 (32/4 = 8)
32
32
  var<workgroup> wg_sums_cols8: array<f32, 64>;
33
33
 
34
34
  struct Uniforms {
@@ -166,10 +166,9 @@ fn main(
166
166
  // Tune-time overrides can retile this without changing entry points.
167
167
  override MULTICOL_COLS_PER_WG: u32 = 32u;
168
168
  override MULTICOL_THREADS_PER_COL: u32 = 8u;
169
- const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
170
169
 
171
170
  // Shared memory for reduction (one slot per thread)
172
- var<workgroup> multicol_wg_sums: array<f32, MAX_WORKGROUP_SIZE>;
171
+ var<workgroup> multicol_wg_sums: array<f16, MAX_WORKGROUP_SIZE>;
173
172
 
174
173
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
175
174
  fn main_multicol(
@@ -193,7 +192,7 @@ fn main_multicol(
193
192
  // Track validity
194
193
  let is_valid = col < u.N;
195
194
 
196
- var partial_sum: f32 = 0.0;
195
+ var partial_sum: f16 = f16(0.0);
197
196
 
198
197
  if (is_valid) {
199
198
  // Each of 8 threads splits K
@@ -218,30 +217,30 @@ fn main_multicol(
218
217
  B[b_row_offset + k + 2u],
219
218
  B[b_row_offset + k + 3u]
220
219
  );
221
- partial_sum += f32(dot(a_vec, b_vec));
220
+ partial_sum += dot(a_vec, b_vec);
222
221
  }
223
222
 
224
223
  for (; k < k_end; k = k + 1u) {
225
- partial_sum += f32(A[k]) * f32(B[b_row_offset + k]);
224
+ partial_sum += A[k] * B[b_row_offset + k];
226
225
  }
227
226
  } else {
228
227
  // B is [K, N] (GGUF/column-major): B[k, col] = B[k * N + col]
229
228
  for (; k < k_aligned_end; k = k + 4u) {
230
- let a0 = f32(A[k]);
231
- let a1 = f32(A[k + 1u]);
232
- let a2 = f32(A[k + 2u]);
233
- let a3 = f32(A[k + 3u]);
229
+ let a0 = A[k];
230
+ let a1 = A[k + 1u];
231
+ let a2 = A[k + 2u];
232
+ let a3 = A[k + 3u];
234
233
 
235
- let b0 = f32(B[k * u.N + col]);
236
- let b1 = f32(B[(k + 1u) * u.N + col]);
237
- let b2 = f32(B[(k + 2u) * u.N + col]);
238
- let b3 = f32(B[(k + 3u) * u.N + col]);
234
+ let b0 = B[k * u.N + col];
235
+ let b1 = B[(k + 1u) * u.N + col];
236
+ let b2 = B[(k + 2u) * u.N + col];
237
+ let b3 = B[(k + 3u) * u.N + col];
239
238
 
240
239
  partial_sum = partial_sum + a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3;
241
240
  }
242
241
 
243
242
  for (; k < k_end; k = k + 1u) {
244
- partial_sum = partial_sum + f32(A[k]) * f32(B[k * u.N + col]);
243
+ partial_sum = partial_sum + A[k] * B[k * u.N + col];
245
244
  }
246
245
  }
247
246
  }
@@ -252,12 +251,12 @@ fn main_multicol(
252
251
 
253
252
  // Thread 0 of each column reduces its 8 values
254
253
  if (thread_in_col == 0u && is_valid) {
255
- var final_sum: f32 = 0.0;
254
+ var final_sum: f16 = f16(0.0);
256
255
  let base = col_in_wg * MULTICOL_THREADS_PER_COL;
257
256
  for (var i: u32 = 0u; i < MULTICOL_THREADS_PER_COL; i = i + 1u) {
258
257
  final_sum = final_sum + multicol_wg_sums[base + i];
259
258
  }
260
- C[col] = f16(final_sum * u.alpha);
259
+ C[col] = final_sum * f16(u.alpha);
261
260
  }
262
261
  }
263
262
 
@@ -442,8 +441,8 @@ fn main_cols8(
442
441
  // giving large contiguous reads for better cache utilization.
443
442
  // Ideal for standard projections (N=1536..8960) where K is moderate.
444
443
 
445
- const COLS_PER_WG_4T: u32 = 64u;
446
- const THREADS_PER_COL_4T: u32 = 4u;
444
+ override COLS_PER_WG_4T: u32 = 64u;
445
+ override THREADS_PER_COL_4T: u32 = 4u;
447
446
  var<workgroup> wg_sums_4t: array<f32, MAX_WORKGROUP_SIZE>;
448
447
 
449
448
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/modulate.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Modulate kernel (f16)
4
4
  // Applies per-channel affine and optional gating.
5
5
 
@@ -37,18 +37,6 @@ export declare function runTopK(
37
37
  options?: MoEOptions
38
38
  ): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
39
39
 
40
- /**
41
- * Record top-K expert selection (batched, no submit)
42
- */
43
- export declare function recordTopK(
44
- recorder: CommandRecorder,
45
- probs: GPUBuffer,
46
- numTokens: number,
47
- numExperts: number,
48
- topK: number,
49
- options?: MoEOptions
50
- ): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
51
-
52
40
  /**
53
41
  * Run MoE gather (dispatch tokens to experts)
54
42
  */
@@ -62,20 +50,6 @@ export declare function runMoEGather(
62
50
  options?: MoEOptions
63
51
  ): Promise<MoEGatherResult>;
64
52
 
65
- /**
66
- * Record MoE gather (batched, no submit)
67
- */
68
- export declare function recordMoEGather(
69
- recorder: CommandRecorder,
70
- hiddenStates: Tensor,
71
- expertIndices: GPUBuffer,
72
- numTokens: number,
73
- hiddenSize: number,
74
- numExperts: number,
75
- topK: number,
76
- options?: MoEOptions
77
- ): Promise<MoEGatherResult>;
78
-
79
53
  /**
80
54
  * Build token offsets for dynamic scatter on GPU.
81
55
  *
@@ -119,21 +93,6 @@ export declare function runScatterAdd(
119
93
  options?: MoEOptions
120
94
  ): Promise<Tensor>;
121
95
 
122
- /**
123
- * Record scatter-add (batched, no submit)
124
- */
125
- export declare function recordScatterAdd(
126
- recorder: CommandRecorder,
127
- expertOutputs: Tensor,
128
- indices: GPUBuffer,
129
- weights: GPUBuffer,
130
- numTokens: number,
131
- hiddenSize: number,
132
- numExperts: number,
133
- topK: number,
134
- options?: MoEOptions
135
- ): Promise<Tensor>;
136
-
137
96
  /**
138
97
  * Run dynamic scatter-add with token offsets
139
98
  */
@@ -147,18 +106,3 @@ export declare function runScatterAddDynamic(
147
106
  topK: number,
148
107
  options?: MoEOptions
149
108
  ): Promise<Tensor>;
150
-
151
- /**
152
- * Record dynamic scatter-add (batched, no submit)
153
- */
154
- export declare function recordScatterAddDynamic(
155
- recorder: CommandRecorder,
156
- expertOutputs: Tensor,
157
- indices: GPUBuffer,
158
- weights: GPUBuffer,
159
- tokenOffsets: GPUBuffer,
160
- numTokens: number,
161
- hiddenSize: number,
162
- topK: number,
163
- options?: MoEOptions
164
- ): Promise<Tensor>;
@@ -115,10 +115,6 @@ export async function runTopK(probs, numTokens, numExperts, topK, options = {})
115
115
  }
116
116
 
117
117
 
118
- export async function recordTopK(recorder, probs, numTokens, numExperts, topK, options = {}) {
119
- return executeTopK(recorder, probs, numTokens, numExperts, topK, options);
120
- }
121
-
122
118
  // Cached explicit bind group layout for MoE gather (all 6 bindings).
123
119
  // Internal postmortems cover why this explicit layout is required.
124
120
 
@@ -312,10 +308,6 @@ export async function runMoEGather(hiddenStates, expertIndices, numTokens, hidde
312
308
  }
313
309
 
314
310
 
315
- export async function recordMoEGather(recorder, hiddenStates, expertIndices, numTokens, hiddenSize, numExperts, topK, options = {}) {
316
- return executeMoEGather(recorder, hiddenStates, expertIndices, numTokens, hiddenSize, numExperts, topK, options);
317
- }
318
-
319
311
  async function executeScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options = {}) {
320
312
  const execution = resolveExecution(recorder);
321
313
  const { outputBuffer = null } = options;
@@ -384,10 +376,6 @@ export async function runScatterAdd(expertOutputs, indices, weights, numTokens,
384
376
  }
385
377
 
386
378
 
387
- export async function recordScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options = {}) {
388
- return executeScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options);
389
- }
390
-
391
379
  async function executeMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap, numTokens, numExperts, topK, maxTokensPerExpert, options = {}) {
392
380
  const execution = resolveExecution(recorder);
393
381
  const { outputBuffer = null } = options;
@@ -537,6 +525,3 @@ export async function runScatterAddDynamic(expertOutputs, indices, weights, toke
537
525
  }
538
526
 
539
527
 
540
- export async function recordScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
541
- return executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
542
- }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/moe_gather.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // moe_gather_f16.wgsl
4
4
 
5
5
  /**
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/pixel_shuffle.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Pixel shuffle (tokens -> CHW) f16
4
4
 
5
5
  enable f16;
@@ -19,18 +19,6 @@ export interface ResidualOptions extends OutputBufferOptions {
19
19
  executionPolicies?: ExecutionV1PoliciesSchema | null;
20
20
  }
21
21
 
22
- /** Context for residual variant selection rules. */
23
- export interface ResidualSelectionContext {
24
- outputDtype: 'f16' | 'f32';
25
- useVec4: boolean;
26
- }
27
-
28
- /** Context for bias add variant selection rules. */
29
- export interface BiasAddSelectionContext {
30
- dataDtype: 'f16' | 'f32';
31
- biasDtype: 'f16' | 'f32';
32
- }
33
-
34
22
  /**
35
23
  * Run residual add (element-wise addition)
36
24
  */
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // residual_f16.wgsl
4
4
 
5
5
  /**
@@ -32,5 +32,5 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
32
32
  if (idx >= u.size) {
33
33
  return;
34
34
  }
35
- output[idx] = f16(f32(a[idx]) + f32(b[idx]));
35
+ output[idx] = a[idx] + b[idx];
36
36
  }
@@ -36,13 +36,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
36
36
  let remaining = min(4u, size - idx);
37
37
 
38
38
  if (remaining >= 4u) {
39
- output[idx] = f16(f32(a[idx]) + f32(b[idx]));
40
- output[idx + 1u] = f16(f32(a[idx + 1u]) + f32(b[idx + 1u]));
41
- output[idx + 2u] = f16(f32(a[idx + 2u]) + f32(b[idx + 2u]));
42
- output[idx + 3u] = f16(f32(a[idx + 3u]) + f32(b[idx + 3u]));
39
+ output[idx] = a[idx] + b[idx];
40
+ output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
41
+ output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
42
+ output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
43
43
  } else {
44
44
  for (var i = 0u; i < remaining; i = i + 1u) {
45
- output[idx + i] = f16(f32(a[idx + i]) + f32(b[idx + i]));
45
+ output[idx + i] = a[idx + i] + b[idx + i];
46
46
  }
47
47
  }
48
48
  }
@@ -18,14 +18,6 @@ export interface RMSNormOptions extends OutputBufferOptions {
18
18
  rmsNormWeightOffset?: boolean;
19
19
  }
20
20
 
21
- /** Context for RMSNorm variant selection rules. */
22
- export interface RMSNormSelectionContext {
23
- isF16: boolean;
24
- residual: boolean;
25
- hasSubgroups: boolean;
26
- isSmall: boolean;
27
- }
28
-
29
21
  /**
30
22
  * Select RMSNorm kernel variant based on options, tensor dtypes, and GPU capabilities.
31
23
  */