doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -3,6 +3,7 @@ import { createTensor } from '../tensor.js';
3
3
  import {
4
4
  getBuffer,
5
5
  getLayout,
6
+ getWeightMetadata,
6
7
  getWeightDtype,
7
8
  isWeightBuffer,
8
9
  resolveWeightBufferMaterialization,
@@ -29,6 +30,7 @@ import {
29
30
  selectMatmulKernel,
30
31
  } from './matmul-selection.js';
31
32
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
33
+ import { getRuntimeConfig } from '../../config/runtime.js';
32
34
  import {
33
35
  MatmulKernel,
34
36
  calculateMatmulDispatch,
@@ -36,6 +38,7 @@ import {
36
38
  createMatmulBindGroupLayout,
37
39
  getMatmulPipeline,
38
40
  } from './matmul-dispatch.js';
41
+ import { RECORD_STAGE_DEBUG_ENABLED, __dbgRecord, getPipelineBindGroupLayout } from './utils.js';
39
42
 
40
43
  export { isFusedQ4KDisabled, selectMatmulKernel };
41
44
  export { createMatmulBindGroupLayout };
@@ -79,6 +82,36 @@ function buildProfileLabel(options = {}) {
79
82
  return `matmul${roleLabel}${layerLabel}`;
80
83
  }
81
84
 
85
+ function resolveLiteRTInt4StorageEncoding(weight, label) {
86
+ const storageEncoding = String(getWeightMetadata(weight)?.storageEncoding ?? '').toLowerCase();
87
+ if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
88
+ throw new Error(
89
+ `[Matmul] LiteRT INT4 weight "${label ?? 'unknown'}" requires metadata.storageEncoding ` +
90
+ `"signed" or "offset_binary", got "${storageEncoding || 'missing'}".`
91
+ );
92
+ }
93
+ return storageEncoding;
94
+ }
95
+
96
+ function resolveW4A16ScaleDtype(weight, label) {
97
+ const metadata = getWeightMetadata(weight);
98
+ const scaleDtype = String(metadata?.scaleDtype ?? '').toLowerCase();
99
+ if (scaleDtype !== 'f16' && scaleDtype !== 'bf16' && scaleDtype !== 'f32') {
100
+ throw new Error(
101
+ `[Matmul] W4A16 weight "${label ?? 'unknown'}" requires metadata.scaleDtype ` +
102
+ `"f16", "bf16", or "f32", got "${scaleDtype || 'missing'}".`
103
+ );
104
+ }
105
+ return scaleDtype;
106
+ }
107
+
108
+ function resolveW4A16ScaleDtypeConstant(scaleDtype) {
109
+ if (scaleDtype === 'f16') return 0;
110
+ if (scaleDtype === 'bf16') return 1;
111
+ if (scaleDtype === 'f32') return 2;
112
+ throw new Error(`[Matmul] unsupported W4A16 scale dtype "${scaleDtype}".`);
113
+ }
114
+
82
115
  function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel, buffer, details = []) {
83
116
  const isGpuBuffer = buffer && (
84
117
  typeof GPUBuffer === 'undefined'
@@ -96,12 +129,23 @@ function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel,
96
129
  );
97
130
  }
98
131
 
99
- function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes) {
132
+ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null, scaleBuffer = null) {
100
133
  const isQ4KF16 = variant === 'q4_fused_multicol_f16'
101
134
  || variant === 'q4_fused_f16a'
102
135
  || variant === 'q4_fused_batched_f16'
103
136
  || variant === 'q4_fused_multicol_f16a'
104
- || variant === 'q4_fused_batched_f16a';
137
+ || variant === 'q4_fused_multicol_f16a_f32acc'
138
+ || variant === 'q4_fused_batched_f16a'
139
+ || variant === 'q4_fused_batched_f16acc_f16a'
140
+ || variant === 'q4_fused_prefill_tiled_f16'
141
+ || variant === 'q4_fused_widetile_f16'
142
+ || variant === 'q4_fused_widetile_f16a';
143
+ // 5-entry Q4K epilogue/prologue variants: output at binding 3 + one
144
+ // extra read-only buffer at binding 4 (residual for _residual, norm weight
145
+ // for _rmsnorm). Distinct from isQ4KF16 (which puts output at binding 4).
146
+ const isQ4KResidual = variant === 'q4_fused_widetile_residual';
147
+ const isWideTileRmsnorm = variant === 'q4_fused_rmsnorm_widetile';
148
+ const isW4A16 = variant.startsWith('w4a16_');
105
149
 
106
150
  assertBindGroupBuffer('matmul', variant, 0, 'uniforms', uniformBuffer);
107
151
  assertBindGroupBuffer('matmul', variant, 1, 'input', matmulInput?.buffer, [
@@ -109,7 +153,22 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
109
153
  `inputDtype=${matmulInput?.dtype ?? 'unknown'}`,
110
154
  ]);
111
155
  assertBindGroupBuffer('matmul', variant, 2, 'weights', bBuffer);
112
- assertBindGroupBuffer('matmul', variant, isQ4KF16 ? 4 : 3, 'output', outputBuffer);
156
+ if (isW4A16) {
157
+ assertBindGroupBuffer('matmul', variant, 3, 'scales', scaleBuffer);
158
+ }
159
+ assertBindGroupBuffer('matmul', variant, (isQ4KF16 || isW4A16) ? 4 : 3, 'output', outputBuffer);
160
+ if (isQ4KResidual) {
161
+ if (!residualBuffer) {
162
+ throw new Error(`[Matmul] variant "${variant}" requires a residual buffer but none was provided.`);
163
+ }
164
+ assertBindGroupBuffer('matmul', variant, 4, 'residual', residualBuffer);
165
+ }
166
+ if (isWideTileRmsnorm) {
167
+ if (!normWeightBuffer) {
168
+ throw new Error(`[Matmul] variant "${variant}" requires a norm weight buffer but none was provided.`);
169
+ }
170
+ assertBindGroupBuffer('matmul', variant, 4, 'norm_weight', normWeightBuffer);
171
+ }
113
172
 
114
173
  const entries = [
115
174
  { binding: 0, resource: { buffer: uniformBuffer } },
@@ -117,7 +176,16 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
117
176
  { binding: 2, resource: { buffer: bBuffer, offset: offsets.bOffset, size: bindingSizes.bBindingSize } },
118
177
  ];
119
178
 
120
- if (isQ4KF16) {
179
+ if (isW4A16) {
180
+ entries.push({
181
+ binding: 3,
182
+ resource: { buffer: scaleBuffer },
183
+ });
184
+ entries.push({
185
+ binding: 4,
186
+ resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
187
+ });
188
+ } else if (isQ4KF16) {
121
189
  entries.push({
122
190
  binding: 4,
123
191
  resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
@@ -127,6 +195,18 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
127
195
  binding: 3,
128
196
  resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
129
197
  });
198
+ if (isQ4KResidual) {
199
+ entries.push({
200
+ binding: 4,
201
+ resource: { buffer: residualBuffer },
202
+ });
203
+ }
204
+ if (isWideTileRmsnorm) {
205
+ entries.push({
206
+ binding: 4,
207
+ resource: { buffer: normWeightBuffer },
208
+ });
209
+ }
130
210
  }
131
211
 
132
212
  return entries;
@@ -214,7 +294,25 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
214
294
 
215
295
  validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
216
296
 
217
- const { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
297
+ const runtimeSession = getRuntimeConfig().inference?.session;
298
+ const effectiveOptions = (
299
+ options.useTiledQ4KPrefill == null
300
+ || options.useWideTileQ4KPrefill == null
301
+ || options.useWideTileQ4KDecode == null
302
+ || options.useWideTileResidualFusion == null
303
+ || options.useFusedRmsnormWideTile == null
304
+ )
305
+ ? {
306
+ ...options,
307
+ useTiledQ4KPrefill: options.useTiledQ4KPrefill ?? (runtimeSession?.useTiledQ4KPrefill === true),
308
+ useWideTileQ4KPrefill: options.useWideTileQ4KPrefill ?? (runtimeSession?.useWideTileQ4KPrefill === true),
309
+ useWideTileQ4KDecode: options.useWideTileQ4KDecode ?? (runtimeSession?.useWideTileQ4KDecode === true),
310
+ useWideTileResidualFusion: options.useWideTileResidualFusion ?? (runtimeSession?.useWideTileResidualFusion === true),
311
+ useFusedRmsnormWideTile: options.useFusedRmsnormWideTile ?? (runtimeSession?.useFusedRmsnormWideTile === true),
312
+ }
313
+ : options;
314
+
315
+ let { variant, useQ4KFused, useGemv, useLiteRTInt4Fused = false, useW4A16Fused = false } = selectMatmulVariantAndFlags(
218
316
  mode,
219
317
  M,
220
318
  N,
@@ -223,10 +321,55 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
223
321
  bDtype,
224
322
  transposeB,
225
323
  requestedOutputDtype,
226
- options
324
+ effectiveOptions
227
325
  );
228
326
 
229
- const constants = resolveMatmulConstants(options, phase);
327
+ if (
328
+ runtimeSession?.useF32AccumF16ioMatmul === true
329
+ && useQ4KFused
330
+ && variant === 'q4_fused_multicol_f16a'
331
+ ) {
332
+ variant = 'q4_fused_multicol_f16a_f32acc';
333
+ }
334
+
335
+ let constants = resolveMatmulConstants(options, phase);
336
+ if (variant === 'f32' && constants && options.constants == null) {
337
+ constants = null;
338
+ }
339
+ if (bDtype === 'litert_int4') {
340
+ const storageEncoding = resolveLiteRTInt4StorageEncoding(resolvedWeight, weightLabel);
341
+ constants = {
342
+ ...(constants ?? {}),
343
+ STORAGE_OFFSET_BINARY: storageEncoding === 'offset_binary' ? 1 : 0,
344
+ };
345
+ }
346
+ let w4a16ScaleBuffer = null;
347
+ if (bDtype === 'w4a16') {
348
+ const metadata = getWeightMetadata(resolvedWeight);
349
+ w4a16ScaleBuffer = metadata?.scaleBuffer ?? null;
350
+ const scaleDtype = resolveW4A16ScaleDtype(resolvedWeight, weightLabel);
351
+ constants = {
352
+ ...(constants ?? {}),
353
+ SCALE_DTYPE: resolveW4A16ScaleDtypeConstant(scaleDtype),
354
+ };
355
+ }
356
+ // For the rmsnorm-fused WideTile variant, forward the caller's
357
+ // rmsNormOffset flag as a pipeline override constant. Gemma-family norm
358
+ // weights encode `(w - 1.0)`; other models encode `w`.
359
+ // Also forward WEIGHT_IS_F16 based on the norm weight buffer dtype so the
360
+ // kernel correctly unpacks f16-packed weights (Gemma hidden weights) vs
361
+ // f32 weights.
362
+ if (variant === 'q4_fused_rmsnorm_widetile' && options.rmsNormOffset != null) {
363
+ const normWeightDtype = getWeightDtype(options.normWeight);
364
+ if (!normWeightDtype) {
365
+ throw new Error('[Matmul] q4_fused_rmsnorm_widetile requires normWeight dtype metadata.');
366
+ }
367
+ constants = {
368
+ ...(constants ?? {}),
369
+ RMS_NORM_OFFSET: options.rmsNormOffset === true,
370
+ WEIGHT_IS_F16: normWeightDtype === 'f16',
371
+ };
372
+ }
230
373
 
231
374
  let matmulInput = A;
232
375
  let matmulADtype = aDtype;
@@ -295,12 +438,24 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
295
438
  log.warn('MatmulQKVProbe', `${err.message} | ${probeDetail}`);
296
439
  }
297
440
  if (shouldFailOnSmallAttentionWeightBuffer) {
441
+ if (!isRecord && castedInput) {
442
+ releaseBuffer(castedInput.buffer);
443
+ castedInput = null;
444
+ }
298
445
  throw new Error(`${err.message}${detail}`);
299
446
  }
300
447
  }
301
448
  if (err instanceof Error && err.message.includes('B buffer too small')) {
449
+ if (!isRecord && castedInput) {
450
+ releaseBuffer(castedInput.buffer);
451
+ castedInput = null;
452
+ }
302
453
  throw new Error(`${err.message}${detail}`);
303
454
  }
455
+ if (!isRecord && castedInput) {
456
+ releaseBuffer(castedInput.buffer);
457
+ castedInput = null;
458
+ }
304
459
  throw err;
305
460
  }
306
461
 
@@ -325,39 +480,72 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
325
480
  );
326
481
  }
327
482
 
328
- const config = getMatmulConfig(variant, constants);
329
- const kernel = new MatmulKernel(device);
330
- const pipeline = await getMatmulPipeline(variant, constants);
331
-
332
- const { output: C, outputSize, cBindingSize, actualOutputDtype } = resolveMatmulOutput(
333
- variant,
334
- M,
335
- N,
336
- outputBuffer
337
- );
338
- const ownsOutput = outputBuffer == null;
339
-
340
- if (isAttnProj && shouldLogAttentionWeightBuffer) {
341
- log.warn('MatmulVariantDiag',
342
- `role=${options.role ?? ''} layer=${Number.isFinite(options.layerIdx) ? options.layerIdx : '?'} mode=${mode} ` +
343
- `variant=${variant} useQ4KFused=${useQ4KFused} useGemv=${useGemv} ` +
344
- `aDtype=${aDtype} bDtype=${bDtype} output=${actualOutputDtype}`
483
+ let __dbg = false;
484
+ let __t0 = 0;
485
+ let __tPipeline = 0;
486
+ let config;
487
+ let kernel;
488
+ let pipeline;
489
+ let C = null;
490
+ let outputSize;
491
+ let cBindingSize;
492
+ let actualOutputDtype;
493
+ let ownsOutput = false;
494
+ let dispatchPlan;
495
+ try {
496
+ __dbg = RECORD_STAGE_DEBUG_ENABLED;
497
+ __t0 = __dbg ? performance.now() : 0;
498
+ config = getMatmulConfig(variant, constants);
499
+ kernel = new MatmulKernel(device);
500
+ pipeline = await getMatmulPipeline(variant, constants);
501
+ __tPipeline = __dbg ? performance.now() : 0;
502
+
503
+ const outputInfo = resolveMatmulOutput(
504
+ variant,
505
+ M,
506
+ N,
507
+ outputBuffer
345
508
  );
346
- }
509
+ C = outputInfo.output;
510
+ outputSize = outputInfo.outputSize;
511
+ cBindingSize = outputInfo.cBindingSize;
512
+ actualOutputDtype = outputInfo.actualOutputDtype;
513
+ ownsOutput = outputBuffer == null;
514
+
515
+ if (isAttnProj && shouldLogAttentionWeightBuffer) {
516
+ log.warn('MatmulVariantDiag',
517
+ `role=${options.role ?? ''} layer=${Number.isFinite(options.layerIdx) ? options.layerIdx : '?'} mode=${mode} ` +
518
+ `variant=${variant} useQ4KFused=${useQ4KFused} useGemv=${useGemv} useLiteRTInt4Fused=${useLiteRTInt4Fused} ` +
519
+ `aDtype=${aDtype} bDtype=${bDtype} output=${actualOutputDtype}`
520
+ );
521
+ }
347
522
 
348
- if (!Number.isFinite(outputSize) || outputSize <= 0) {
349
- throw new Error(`[${opLabel}] Invalid output size: ${outputSize} (M=${M}, N=${N})`);
350
- }
523
+ if (!Number.isFinite(outputSize) || outputSize <= 0) {
524
+ throw new Error(`[${opLabel}] Invalid output size: ${outputSize} (M=${M}, N=${N})`);
525
+ }
351
526
 
352
- const cRequired = cOffset + cBindingSize;
353
- if (C.size < cRequired) {
354
- throw new Error(`[${opLabel}] Output buffer too small: ${C.size} < ${cRequired} (M=${M}, N=${N})`);
355
- }
527
+ const cRequired = cOffset + cBindingSize;
528
+ if (C.size < cRequired) {
529
+ throw new Error(`[${opLabel}] Output buffer too small: ${C.size} < ${cRequired} (M=${M}, N=${N})`);
530
+ }
356
531
 
357
- const dispatchPlan = calculateMatmulDispatch(variant, useQ4KFused, useGemv, M, N, config);
532
+ dispatchPlan = calculateMatmulDispatch(variant, useQ4KFused, useGemv, useLiteRTInt4Fused, M, N, config, useW4A16Fused);
533
+ } catch (error) {
534
+ if (!isRecord && castedInput) {
535
+ releaseBuffer(castedInput.buffer);
536
+ castedInput = null;
537
+ }
538
+ if (ownsOutput && C) {
539
+ releaseBuffer(C);
540
+ }
541
+ throw error;
542
+ }
358
543
  let uniformBuffer = null;
359
544
  let completed = false;
360
545
  try {
546
+ const uniformExtras = variant === 'q4_fused_rmsnorm_widetile' && Number.isFinite(options.rmsNormEps)
547
+ ? { eps: options.rmsNormEps }
548
+ : null;
361
549
  uniformBuffer = createMatmulUniformBuffer(
362
550
  'matmul_uniforms',
363
551
  M,
@@ -368,9 +556,12 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
368
556
  transposeB,
369
557
  dispatchPlan.uniformWorkgroupsX,
370
558
  recorder || null,
371
- device
559
+ device,
560
+ uniformExtras
372
561
  );
373
562
 
563
+ const residualBuffer = options.residualTensor?.buffer ?? null;
564
+ const normWeightBuffer = options.normWeight?.buffer ?? options.normWeight ?? null;
374
565
  const entries = createMatmulBindGroupEntries(
375
566
  variant,
376
567
  uniformBuffer,
@@ -382,30 +573,41 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
382
573
  aBindingSize: bindingSizes.aBindingSize,
383
574
  bBindingSize: bindingSizes.bBindingSize,
384
575
  cBindingSize,
385
- }
576
+ },
577
+ residualBuffer,
578
+ normWeightBuffer,
579
+ w4a16ScaleBuffer
386
580
  );
387
581
 
582
+ const __tBgStart = __dbg ? performance.now() : 0;
388
583
  const bindGroup = device.createBindGroup({
389
584
  label: 'matmul_bind_group',
390
- layout: pipeline.getBindGroupLayout(0),
585
+ layout: getPipelineBindGroupLayout(pipeline, 0),
391
586
  entries,
392
587
  });
588
+ const __tBg = __dbg ? performance.now() : 0;
393
589
 
394
590
  if (isRecord) {
395
591
  kernel.record(recorder, pipeline, bindGroup, dispatchPlan.workgroups, buildProfileLabel(options));
396
592
  } else {
397
593
  kernel.dispatch(pipeline, bindGroup, dispatchPlan.workgroups);
398
594
  }
595
+ if (__dbg) {
596
+ const __tEnd = performance.now();
597
+ __dbgRecord('matmul', variant, __tPipeline - __t0, __tBgStart - __tPipeline, __tBg - __tBgStart, __tEnd - __tBg);
598
+ }
599
+ const tensor = createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
399
600
  completed = true;
400
- return createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
601
+ return tensor;
401
602
  } finally {
402
603
  if (!isRecord && uniformBuffer) {
403
604
  releaseUniformBuffer(uniformBuffer);
404
605
  }
405
606
  if (!isRecord && castedInput) {
406
607
  releaseBuffer(castedInput.buffer);
608
+ castedInput = null;
407
609
  }
408
- if (!completed && ownsOutput) {
610
+ if (!completed && ownsOutput && C) {
409
611
  releaseBuffer(C);
410
612
  }
411
613
  }
@@ -8,7 +8,7 @@
8
8
  // C[M,N] = A[M,K] * B^T[N,K] (transpose_b=1, SafeTensors/RDRR row-major layout)
9
9
  //
10
10
  // Design:
11
- // - f16 storage, f32 accumulators for numerical stability
11
+ // - f16 storage and f16 accumulators for the all-f16 execution lane
12
12
  // - coalesced scalar loads into shared tiles
13
13
  // - Shared memory tiles: 64 x TILE_K for A and B
14
14
  // - B is always assumed row-major (transpose_b=1) for RDRR models
@@ -64,11 +64,10 @@ fn main(
64
64
  let row_base = wg_id.x * TILE_M + tx * THREAD_M;
65
65
  let col_base = wg_id.y * TILE_N + ty * THREAD_N;
66
66
 
67
- // f32 accumulators: 4x4 = 16 per thread
68
- var acc00: f32 = 0.0; var acc01: f32 = 0.0; var acc02: f32 = 0.0; var acc03: f32 = 0.0;
69
- var acc10: f32 = 0.0; var acc11: f32 = 0.0; var acc12: f32 = 0.0; var acc13: f32 = 0.0;
70
- var acc20: f32 = 0.0; var acc21: f32 = 0.0; var acc22: f32 = 0.0; var acc23: f32 = 0.0;
71
- var acc30: f32 = 0.0; var acc31: f32 = 0.0; var acc32: f32 = 0.0; var acc33: f32 = 0.0;
67
+ var acc00: f16 = f16(0.0); var acc01: f16 = f16(0.0); var acc02: f16 = f16(0.0); var acc03: f16 = f16(0.0);
68
+ var acc10: f16 = f16(0.0); var acc11: f16 = f16(0.0); var acc12: f16 = f16(0.0); var acc13: f16 = f16(0.0);
69
+ var acc20: f16 = f16(0.0); var acc21: f16 = f16(0.0); var acc22: f16 = f16(0.0); var acc23: f16 = f16(0.0);
70
+ var acc30: f16 = f16(0.0); var acc31: f16 = f16(0.0); var acc32: f16 = f16(0.0); var acc33: f16 = f16(0.0);
72
71
 
73
72
  let num_tiles = (u.K + TILE_K - 1u) / TILE_K;
74
73
 
@@ -120,16 +119,16 @@ fn main(
120
119
  // B tile: tileB[n_local * TILE_K + k] for cols ty*4..ty*4+3
121
120
  for (var k: u32 = 0u; k < TILE_K; k = k + 1u) {
122
121
  // Load 4 A values for this thread's rows
123
- let a0 = f32(tileA[(tx * THREAD_M + 0u) * TILE_K + k]);
124
- let a1 = f32(tileA[(tx * THREAD_M + 1u) * TILE_K + k]);
125
- let a2 = f32(tileA[(tx * THREAD_M + 2u) * TILE_K + k]);
126
- let a3 = f32(tileA[(tx * THREAD_M + 3u) * TILE_K + k]);
122
+ let a0 = tileA[(tx * THREAD_M + 0u) * TILE_K + k];
123
+ let a1 = tileA[(tx * THREAD_M + 1u) * TILE_K + k];
124
+ let a2 = tileA[(tx * THREAD_M + 2u) * TILE_K + k];
125
+ let a3 = tileA[(tx * THREAD_M + 3u) * TILE_K + k];
127
126
 
128
127
  // Load 4 B values for this thread's columns
129
- let b0 = f32(tileB[(ty * THREAD_N + 0u) * TILE_K + k]);
130
- let b1 = f32(tileB[(ty * THREAD_N + 1u) * TILE_K + k]);
131
- let b2 = f32(tileB[(ty * THREAD_N + 2u) * TILE_K + k]);
132
- let b3 = f32(tileB[(ty * THREAD_N + 3u) * TILE_K + k]);
128
+ let b0 = tileB[(ty * THREAD_N + 0u) * TILE_K + k];
129
+ let b1 = tileB[(ty * THREAD_N + 1u) * TILE_K + k];
130
+ let b2 = tileB[(ty * THREAD_N + 2u) * TILE_K + k];
131
+ let b3 = tileB[(ty * THREAD_N + 3u) * TILE_K + k];
133
132
 
134
133
  // Outer product: 4x4 accumulation
135
134
  acc00 += a0 * b0; acc01 += a0 * b1; acc02 += a0 * b2; acc03 += a0 * b3;
@@ -142,24 +141,24 @@ fn main(
142
141
  }
143
142
 
144
143
  // Write 4x4 output tile
145
- let alpha = u.alpha;
146
- if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = f16(acc00 * alpha); }
147
- if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = f16(acc01 * alpha); }
148
- if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = f16(acc02 * alpha); }
149
- if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = f16(acc03 * alpha); }
150
-
151
- if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = f16(acc10 * alpha); }
152
- if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = f16(acc11 * alpha); }
153
- if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = f16(acc12 * alpha); }
154
- if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = f16(acc13 * alpha); }
155
-
156
- if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = f16(acc20 * alpha); }
157
- if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = f16(acc21 * alpha); }
158
- if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = f16(acc22 * alpha); }
159
- if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = f16(acc23 * alpha); }
160
-
161
- if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = f16(acc30 * alpha); }
162
- if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = f16(acc31 * alpha); }
163
- if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = f16(acc32 * alpha); }
164
- if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = f16(acc33 * alpha); }
144
+ let alpha = f16(u.alpha);
145
+ if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = acc00 * alpha; }
146
+ if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = acc01 * alpha; }
147
+ if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = acc02 * alpha; }
148
+ if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = acc03 * alpha; }
149
+
150
+ if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = acc10 * alpha; }
151
+ if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = acc11 * alpha; }
152
+ if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = acc12 * alpha; }
153
+ if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = acc13 * alpha; }
154
+
155
+ if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = acc20 * alpha; }
156
+ if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = acc21 * alpha; }
157
+ if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = acc22 * alpha; }
158
+ if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = acc23 * alpha; }
159
+
160
+ if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = acc30 * alpha; }
161
+ if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = acc31 * alpha; }
162
+ if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = acc32 * alpha; }
163
+ if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = acc33 * alpha; }
165
164
  }
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/matmul_gemv_residual.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused GEMV + Residual Kernel - Single dispatch for output projection + residual add
4
4
  // For M=1 decode: C[N] = A[K] * B^T[K,N] + residual[N]
5
5
  //
@@ -152,7 +152,6 @@ fn main(
152
152
  // Tune-time overrides can retile this without changing entry points.
153
153
  override MULTICOL_COLS_PER_WG: u32 = 32u;
154
154
  override MULTICOL_THREADS_PER_COL: u32 = 8u;
155
- const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
156
155
 
157
156
  // Shared memory for reduction (one slot per thread)
158
157
  var<workgroup> multicol_wg_sums: array<f32, MAX_WORKGROUP_SIZE>;
@@ -8,7 +8,7 @@
8
8
  // 4. Loop unrolling for better ILP
9
9
  //
10
10
  // A is f16 (activations), B is f16 (weights), C is f16.
11
- // Accumulation is performed in f32 for stability.
11
+ // The Gemma 4 31B all-f16 lane uses f16 accumulation on main_multicol.
12
12
  // transpose_b=0: B is [K, N] (GGUF/column-major), access B[k * N + col]
13
13
  // transpose_b=1: B is [N, K] (SafeTensors/row-major), access B[col * K + k]
14
14
  //
@@ -20,15 +20,15 @@ enable subgroups;
20
20
 
21
21
  override WORKGROUP_SIZE: u32 = 256u;
22
22
  const MAX_WORKGROUP_SIZE: u32 = 256u;
23
- const COLS_PER_WG: u32 = 4u; // Each workgroup computes 4 output columns
24
- const THREADS_PER_COL: u32 = 64u; // 256 / 4 = 64 threads per column
25
- const MAX_SUBGROUPS_PER_COL: u32 = 16u; // Support sg_size >= 4 (64/4 = 16)
23
+ override COLS_PER_WG: u32 = 4u; // Each workgroup computes 4 output columns
24
+ override THREADS_PER_COL: u32 = 64u; // 256 / 4 = 64 threads per column
25
+ override MAX_SUBGROUPS_PER_COL: u32 = 16u; // Support sg_size >= 4 (64/4 = 16)
26
26
 
27
27
  // Alternative layout: more columns per workgroup, fewer threads per column.
28
28
  // This reduces workgroup count and improves occupancy for mid-sized K.
29
- const COLS_PER_WG_COLS8: u32 = 8u; // 256 / 8 = 32 threads per column
30
- const THREADS_PER_COL_COLS8: u32 = 32u;
31
- const MAX_SUBGROUPS_PER_COL_COLS8: u32 = 8u; // Support sg_size >= 4 (32/4 = 8)
29
+ override COLS_PER_WG_COLS8: u32 = 8u; // 256 / 8 = 32 threads per column
30
+ override THREADS_PER_COL_COLS8: u32 = 32u;
31
+ override MAX_SUBGROUPS_PER_COL_COLS8: u32 = 8u; // Support sg_size >= 4 (32/4 = 8)
32
32
  var<workgroup> wg_sums_cols8: array<f32, 64>;
33
33
 
34
34
  struct Uniforms {
@@ -166,10 +166,9 @@ fn main(
166
166
  // Tune-time overrides can retile this without changing entry points.
167
167
  override MULTICOL_COLS_PER_WG: u32 = 32u;
168
168
  override MULTICOL_THREADS_PER_COL: u32 = 8u;
169
- const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
170
169
 
171
170
  // Shared memory for reduction (one slot per thread)
172
- var<workgroup> multicol_wg_sums: array<f32, MAX_WORKGROUP_SIZE>;
171
+ var<workgroup> multicol_wg_sums: array<f16, MAX_WORKGROUP_SIZE>;
173
172
 
174
173
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
175
174
  fn main_multicol(
@@ -193,7 +192,7 @@ fn main_multicol(
193
192
  // Track validity
194
193
  let is_valid = col < u.N;
195
194
 
196
- var partial_sum: f32 = 0.0;
195
+ var partial_sum: f16 = f16(0.0);
197
196
 
198
197
  if (is_valid) {
199
198
  // Each of 8 threads splits K
@@ -218,30 +217,30 @@ fn main_multicol(
218
217
  B[b_row_offset + k + 2u],
219
218
  B[b_row_offset + k + 3u]
220
219
  );
221
- partial_sum += f32(dot(a_vec, b_vec));
220
+ partial_sum += dot(a_vec, b_vec);
222
221
  }
223
222
 
224
223
  for (; k < k_end; k = k + 1u) {
225
- partial_sum += f32(A[k]) * f32(B[b_row_offset + k]);
224
+ partial_sum += A[k] * B[b_row_offset + k];
226
225
  }
227
226
  } else {
228
227
  // B is [K, N] (GGUF/column-major): B[k, col] = B[k * N + col]
229
228
  for (; k < k_aligned_end; k = k + 4u) {
230
- let a0 = f32(A[k]);
231
- let a1 = f32(A[k + 1u]);
232
- let a2 = f32(A[k + 2u]);
233
- let a3 = f32(A[k + 3u]);
229
+ let a0 = A[k];
230
+ let a1 = A[k + 1u];
231
+ let a2 = A[k + 2u];
232
+ let a3 = A[k + 3u];
234
233
 
235
- let b0 = f32(B[k * u.N + col]);
236
- let b1 = f32(B[(k + 1u) * u.N + col]);
237
- let b2 = f32(B[(k + 2u) * u.N + col]);
238
- let b3 = f32(B[(k + 3u) * u.N + col]);
234
+ let b0 = B[k * u.N + col];
235
+ let b1 = B[(k + 1u) * u.N + col];
236
+ let b2 = B[(k + 2u) * u.N + col];
237
+ let b3 = B[(k + 3u) * u.N + col];
239
238
 
240
239
  partial_sum = partial_sum + a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3;
241
240
  }
242
241
 
243
242
  for (; k < k_end; k = k + 1u) {
244
- partial_sum = partial_sum + f32(A[k]) * f32(B[k * u.N + col]);
243
+ partial_sum = partial_sum + A[k] * B[k * u.N + col];
245
244
  }
246
245
  }
247
246
  }
@@ -252,12 +251,12 @@ fn main_multicol(
252
251
 
253
252
  // Thread 0 of each column reduces its 8 values
254
253
  if (thread_in_col == 0u && is_valid) {
255
- var final_sum: f32 = 0.0;
254
+ var final_sum: f16 = f16(0.0);
256
255
  let base = col_in_wg * MULTICOL_THREADS_PER_COL;
257
256
  for (var i: u32 = 0u; i < MULTICOL_THREADS_PER_COL; i = i + 1u) {
258
257
  final_sum = final_sum + multicol_wg_sums[base + i];
259
258
  }
260
- C[col] = f16(final_sum * u.alpha);
259
+ C[col] = final_sum * f16(u.alpha);
261
260
  }
262
261
  }
263
262
 
@@ -442,8 +441,8 @@ fn main_cols8(
442
441
  // giving large contiguous reads for better cache utilization.
443
442
  // Ideal for standard projections (N=1536..8960) where K is moderate.
444
443
 
445
- const COLS_PER_WG_4T: u32 = 64u;
446
- const THREADS_PER_COL_4T: u32 = 4u;
444
+ override COLS_PER_WG_4T: u32 = 64u;
445
+ override THREADS_PER_COL_4T: u32 = 4u;
447
446
  var<workgroup> wg_sums_4t: array<f32, MAX_WORKGROUP_SIZE>;
448
447
 
449
448
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/modulate.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Modulate kernel (f16)
4
4
  // Applies per-channel affine and optional gating.
5
5