doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -0,0 +1,130 @@
1
+ override WORKGROUP_SIZE: u32 = 256u;
2
+
3
+ const NEG_INF: f32 = -3.402823e+38;
4
+ const NO_PAD_TOKEN: u32 = 0xffffffffu;
5
+
6
+ struct Uniforms {
7
+ vocab_size: u32,
8
+ canvas_length: u32,
9
+ pad_token_id: u32,
10
+ _pad0: u32,
11
+ temperature: f32,
12
+ logit_softcap: f32,
13
+ _pad1: f32,
14
+ _pad2: f32,
15
+ }
16
+
17
+ @group(0) @binding(0) var<uniform> u: Uniforms;
18
+ @group(0) @binding(1) var<storage, read> logits: array<f32>;
19
+ @group(0) @binding(2) var<storage, read_write> argmax_tokens: array<u32>;
20
+ @group(0) @binding(3) var<storage, read_write> entropies: array<f32>;
21
+
22
+ var<workgroup> shared_max: array<f32, 256>;
23
+ var<workgroup> shared_indices: array<u32, 256>;
24
+ var<workgroup> shared_sum: array<f32, 256>;
25
+ var<workgroup> shared_weighted: array<f32, 256>;
26
+
27
+ fn apply_softcap(x: f32, softcap: f32) -> f32 {
28
+ if (softcap <= 0.0) {
29
+ return x;
30
+ }
31
+ return softcap * tanh(x / softcap);
32
+ }
33
+
34
+ fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32, best_index: u32) -> bool {
35
+ if (candidate_value > best_value) {
36
+ return true;
37
+ }
38
+ if (candidate_value < best_value) {
39
+ return false;
40
+ }
41
+ return candidate_index < best_index;
42
+ }
43
+
44
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
45
+ fn entropy_stats(
46
+ @builtin(local_invocation_id) local_id: vec3<u32>,
47
+ @builtin(workgroup_id) workgroup_id: vec3<u32>
48
+ ) {
49
+ let row = workgroup_id.x;
50
+ let thread_idx = local_id.x;
51
+ if (row >= u.canvas_length) {
52
+ return;
53
+ }
54
+
55
+ let base = row * u.vocab_size;
56
+ let temperature = max(u.temperature, 0.000001);
57
+ let pad_id = u.pad_token_id;
58
+ let softcap = u.logit_softcap;
59
+
60
+ var local_max = NEG_INF;
61
+ var local_index = 0u;
62
+ var idx = thread_idx;
63
+ while (idx < u.vocab_size) {
64
+ if (pad_id == NO_PAD_TOKEN || idx != pad_id) {
65
+ let value = apply_softcap(logits[base + idx], softcap) / temperature;
66
+ if (candidate_beats(value, idx, local_max, local_index)) {
67
+ local_max = value;
68
+ local_index = idx;
69
+ }
70
+ }
71
+ idx = idx + WORKGROUP_SIZE;
72
+ }
73
+
74
+ shared_max[thread_idx] = local_max;
75
+ shared_indices[thread_idx] = local_index;
76
+ workgroupBarrier();
77
+
78
+ var stride = WORKGROUP_SIZE / 2u;
79
+ while (stride > 0u) {
80
+ if (thread_idx < stride) {
81
+ if (candidate_beats(
82
+ shared_max[thread_idx + stride],
83
+ shared_indices[thread_idx + stride],
84
+ shared_max[thread_idx],
85
+ shared_indices[thread_idx]
86
+ )) {
87
+ shared_max[thread_idx] = shared_max[thread_idx + stride];
88
+ shared_indices[thread_idx] = shared_indices[thread_idx + stride];
89
+ }
90
+ }
91
+ workgroupBarrier();
92
+ stride = stride / 2u;
93
+ }
94
+
95
+ let row_max = shared_max[0];
96
+
97
+ var local_sum = 0.0;
98
+ var local_weighted = 0.0;
99
+ idx = thread_idx;
100
+ while (idx < u.vocab_size) {
101
+ if (pad_id == NO_PAD_TOKEN || idx != pad_id) {
102
+ let value = apply_softcap(logits[base + idx], softcap) / temperature;
103
+ let exp_value = exp(value - row_max);
104
+ local_sum = local_sum + exp_value;
105
+ local_weighted = local_weighted + exp_value * value;
106
+ }
107
+ idx = idx + WORKGROUP_SIZE;
108
+ }
109
+
110
+ shared_sum[thread_idx] = local_sum;
111
+ shared_weighted[thread_idx] = local_weighted;
112
+ workgroupBarrier();
113
+
114
+ stride = WORKGROUP_SIZE / 2u;
115
+ while (stride > 0u) {
116
+ if (thread_idx < stride) {
117
+ shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
118
+ shared_weighted[thread_idx] = shared_weighted[thread_idx] + shared_weighted[thread_idx + stride];
119
+ }
120
+ workgroupBarrier();
121
+ stride = stride / 2u;
122
+ }
123
+
124
+ if (thread_idx == 0u) {
125
+ let sum = shared_sum[0];
126
+ let entropy = select(0.0, log(sum) + row_max - (shared_weighted[0] / sum), sum > 0.0);
127
+ argmax_tokens[row] = shared_indices[0];
128
+ entropies[row] = entropy;
129
+ }
130
+ }
@@ -10,6 +10,9 @@ function normalizeWorkgroups(workgroups) {
10
10
  if (!Array.isArray(workgroups) || workgroups.length === 0) {
11
11
  throw new Error('dispatch requires workgroups as a number or [x, y, z]');
12
12
  }
13
+ if (workgroups.length >= 3 && workgroups[1] !== undefined && workgroups[2] !== undefined) {
14
+ return workgroups;
15
+ }
13
16
  const x = workgroups[0] ?? 1;
14
17
  const y = workgroups[1] ?? 1;
15
18
  const z = workgroups[2] ?? 1;
@@ -72,7 +75,12 @@ export function recordDispatch(
72
75
  workgroups,
73
76
  label = 'compute'
74
77
  ) {
75
- const [x, y, z] = assertWorkgroupLimits(recorder.device, workgroups, label);
78
+ const dispatchWorkgroups = assertWorkgroupLimits(recorder.device, workgroups, label);
79
+ if (typeof recorder.recordDispatch === 'function') {
80
+ recorder.recordDispatch(pipeline, bindGroup, dispatchWorkgroups, label);
81
+ return;
82
+ }
83
+ const [x, y, z] = dispatchWorkgroups;
76
84
  const pass = recorder.beginComputePass(label);
77
85
  pass.setPipeline(pipeline);
78
86
  pass.setBindGroup(0, bindGroup);
@@ -108,11 +116,13 @@ export function recordDispatchIndirect(
108
116
  indirectOffset = 0,
109
117
  label = 'compute'
110
118
  ) {
119
+ if (typeof recorder.recordDispatchIndirect === 'function') {
120
+ recorder.recordDispatchIndirect(pipeline, bindGroup, indirectBuffer, indirectOffset, label);
121
+ return;
122
+ }
111
123
  const pass = recorder.beginComputePass(label);
112
124
  pass.setPipeline(pipeline);
113
125
  pass.setBindGroup(0, bindGroup);
114
126
  pass.dispatchWorkgroupsIndirect(indirectBuffer, indirectOffset);
115
127
  pass.end();
116
128
  }
117
-
118
-
@@ -5,7 +5,7 @@ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
5
5
  import { createTensor } from '../tensor.js';
6
6
  import { KernelBase } from './kernel-base.js';
7
7
  import { FFN_DISPATCH } from './constants.js';
8
- import { createUniformBufferWithView } from './utils.js';
8
+ import { createUniformBufferWithView, getPipelineBindGroupLayout } from './utils.js';
9
9
  import { trace, isTraceEnabled } from '../../debug/index.js';
10
10
  import { getBuffer, getWeightDtype } from '../weight-buffer.js';
11
11
  import { isFusedQ4KDisabled } from './matmul.js';
@@ -50,7 +50,6 @@ const F16_OUTPUT_VARIANTS = new Set([
50
50
  'f16_native_batched',
51
51
  ]);
52
52
 
53
-
54
53
  function selectFFNVariant(batchSize, weightDtype, intermediateSize, hiddenSize, inputDtype) {
55
54
  const { multiOutputThreshold } = getKernelThresholds().ffn;
56
55
  const capabilities = getKernelCapabilities();
@@ -227,7 +226,7 @@ export async function runFusedFFN(
227
226
  try {
228
227
  const bindGroup = device.createBindGroup({
229
228
  label: 'fused_ffn_bind_group',
230
- layout: pipeline.getBindGroupLayout(0),
229
+ layout: getPipelineBindGroupLayout(pipeline, 0),
231
230
  entries: [
232
231
  { binding: 0, resource: { buffer: uniformBuffer } },
233
232
  { binding: 1, resource: { buffer: input.buffer } },
@@ -319,7 +318,7 @@ export async function recordFusedFFN(
319
318
  try {
320
319
  const bindGroup = device.createBindGroup({
321
320
  label: 'fused_ffn_bind_group',
322
- layout: pipeline.getBindGroupLayout(0),
321
+ layout: getPipelineBindGroupLayout(pipeline, 0),
323
322
  entries: [
324
323
  { binding: 0, resource: { buffer: uniformBuffer } },
325
324
  { binding: 1, resource: { buffer: input.buffer } },
@@ -0,0 +1,78 @@
1
+ enable f16;
2
+
3
+ override WORKGROUP_SIZE: u32 = 256u;
4
+ override COLS_PER_WG: u32 = 8u;
5
+ override MULTICOL_COLS_PER_WG: u32 = 8u;
6
+ override MULTICOL_THREADS_PER_COL: u32 = 32u;
7
+ override WEIGHT_SCALE: f32 = 0.0625;
8
+ override STORAGE_OFFSET_BINARY: u32 = 0u;
9
+
10
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
11
+
12
+ struct Uniforms {
13
+ M: u32,
14
+ N: u32,
15
+ K: u32,
16
+ alpha: f32,
17
+ transpose_b: u32,
18
+ workgroups_x: u32,
19
+ _pad1: u32,
20
+ _pad2: u32,
21
+ }
22
+
23
+ @group(0) @binding(0) var<uniform> u: Uniforms;
24
+ @group(0) @binding(1) var<storage, read> A: array<f16>;
25
+ @group(0) @binding(2) var<storage, read> B_words: array<u32>;
26
+ @group(0) @binding(3) var<storage, read_write> C_f16: array<f16>;
27
+
28
+ var<workgroup> partials: array<f32, MAX_WORKGROUP_SIZE>;
29
+
30
+ fn packed_byte(byte_index: u32) -> u32 {
31
+ let word = B_words[byte_index >> 2u];
32
+ let shift = (byte_index & 3u) * 8u;
33
+ return (word >> shift) & 0xffu;
34
+ }
35
+
36
+ fn signed_int4(byte_value: u32, high_nibble: bool) -> f32 {
37
+ let value = select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, high_nibble);
38
+ if (STORAGE_OFFSET_BINARY == 1u) {
39
+ return f32(value) - 8.0;
40
+ }
41
+ return select(f32(value), f32(i32(value) - 16), value >= 8u);
42
+ }
43
+
44
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
45
+ fn main_multicol_f16a(
46
+ @builtin(local_invocation_id) lid: vec3<u32>,
47
+ @builtin(workgroup_id) wg_id: vec3<u32>
48
+ ) {
49
+ let local_id = lid.x;
50
+ let lanes_per_col = WORKGROUP_SIZE / MULTICOL_COLS_PER_WG;
51
+ let col_in_wg = local_id / lanes_per_col;
52
+ let lane_in_col = local_id - col_in_wg * lanes_per_col;
53
+ let row = wg_id.y;
54
+ let col = wg_id.x * MULTICOL_COLS_PER_WG + col_in_wg;
55
+ let packed_bytes_per_row = (u.K + 1u) >> 1u;
56
+
57
+ var sum: f32 = 0.0;
58
+ if (row < u.M && col < u.N) {
59
+ let row_base = col * packed_bytes_per_row;
60
+ for (var k: u32 = lane_in_col; k < u.K; k = k + lanes_per_col) {
61
+ let byte_value = packed_byte(row_base + (k >> 1u));
62
+ let weight = signed_int4(byte_value, (k & 1u) == 1u) * WEIGHT_SCALE;
63
+ sum = sum + f32(A[row * u.K + k]) * weight;
64
+ }
65
+ }
66
+
67
+ partials[local_id] = sum;
68
+ workgroupBarrier();
69
+
70
+ if (lane_in_col == 0u && row < u.M && col < u.N) {
71
+ var final_sum: f32 = 0.0;
72
+ let base = col_in_wg * lanes_per_col;
73
+ for (var lane: u32 = 0u; lane < lanes_per_col; lane = lane + 1u) {
74
+ final_sum = final_sum + partials[base + lane];
75
+ }
76
+ C_f16[row * u.N + col] = f16(final_sum * u.alpha);
77
+ }
78
+ }
@@ -0,0 +1,78 @@
1
+ enable f16;
2
+
3
+ override WORKGROUP_SIZE: u32 = 256u;
4
+ override COLS_PER_WG: u32 = 8u;
5
+ override MULTICOL_COLS_PER_WG: u32 = 8u;
6
+ override MULTICOL_THREADS_PER_COL: u32 = 32u;
7
+ override WEIGHT_SCALE: f32 = 0.0625;
8
+ override STORAGE_OFFSET_BINARY: u32 = 0u;
9
+
10
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
11
+
12
+ struct Uniforms {
13
+ M: u32,
14
+ N: u32,
15
+ K: u32,
16
+ alpha: f32,
17
+ transpose_b: u32,
18
+ workgroups_x: u32,
19
+ _pad1: u32,
20
+ _pad2: u32,
21
+ }
22
+
23
+ @group(0) @binding(0) var<uniform> u: Uniforms;
24
+ @group(0) @binding(1) var<storage, read> A: array<f16>;
25
+ @group(0) @binding(2) var<storage, read> B_words: array<u32>;
26
+ @group(0) @binding(3) var<storage, read_write> C_f32: array<f32>;
27
+
28
+ var<workgroup> partials: array<f32, MAX_WORKGROUP_SIZE>;
29
+
30
+ fn packed_byte(byte_index: u32) -> u32 {
31
+ let word = B_words[byte_index >> 2u];
32
+ let shift = (byte_index & 3u) * 8u;
33
+ return (word >> shift) & 0xffu;
34
+ }
35
+
36
+ fn signed_int4(byte_value: u32, high_nibble: bool) -> f32 {
37
+ let value = select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, high_nibble);
38
+ if (STORAGE_OFFSET_BINARY == 1u) {
39
+ return f32(value) - 8.0;
40
+ }
41
+ return select(f32(value), f32(i32(value) - 16), value >= 8u);
42
+ }
43
+
44
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
45
+ fn main_multicol_f16a_f32out(
46
+ @builtin(local_invocation_id) lid: vec3<u32>,
47
+ @builtin(workgroup_id) wg_id: vec3<u32>
48
+ ) {
49
+ let local_id = lid.x;
50
+ let lanes_per_col = WORKGROUP_SIZE / MULTICOL_COLS_PER_WG;
51
+ let col_in_wg = local_id / lanes_per_col;
52
+ let lane_in_col = local_id - col_in_wg * lanes_per_col;
53
+ let row = wg_id.y;
54
+ let col = wg_id.x * MULTICOL_COLS_PER_WG + col_in_wg;
55
+ let packed_bytes_per_row = (u.K + 1u) >> 1u;
56
+
57
+ var sum: f32 = 0.0;
58
+ if (row < u.M && col < u.N) {
59
+ let row_base = col * packed_bytes_per_row;
60
+ for (var k: u32 = lane_in_col; k < u.K; k = k + lanes_per_col) {
61
+ let byte_value = packed_byte(row_base + (k >> 1u));
62
+ let weight = signed_int4(byte_value, (k & 1u) == 1u) * WEIGHT_SCALE;
63
+ sum = sum + f32(A[row * u.K + k]) * weight;
64
+ }
65
+ }
66
+
67
+ partials[local_id] = sum;
68
+ workgroupBarrier();
69
+
70
+ if (lane_in_col == 0u && row < u.M && col < u.N) {
71
+ var final_sum: f32 = 0.0;
72
+ let base = col_in_wg * lanes_per_col;
73
+ for (var lane: u32 = 0u; lane < lanes_per_col; lane = lane + 1u) {
74
+ final_sum = final_sum + partials[base + lane];
75
+ }
76
+ C_f32[row * u.N + col] = final_sum * u.alpha;
77
+ }
78
+ }
@@ -0,0 +1,76 @@
1
+ override WORKGROUP_SIZE: u32 = 256u;
2
+ override COLS_PER_WG: u32 = 8u;
3
+ override MULTICOL_COLS_PER_WG: u32 = 8u;
4
+ override MULTICOL_THREADS_PER_COL: u32 = 32u;
5
+ override WEIGHT_SCALE: f32 = 0.0625;
6
+ override STORAGE_OFFSET_BINARY: u32 = 0u;
7
+
8
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
9
+
10
+ struct Uniforms {
11
+ M: u32,
12
+ N: u32,
13
+ K: u32,
14
+ alpha: f32,
15
+ transpose_b: u32,
16
+ workgroups_x: u32,
17
+ _pad1: u32,
18
+ _pad2: u32,
19
+ }
20
+
21
+ @group(0) @binding(0) var<uniform> u: Uniforms;
22
+ @group(0) @binding(1) var<storage, read> A: array<f32>;
23
+ @group(0) @binding(2) var<storage, read> B_words: array<u32>;
24
+ @group(0) @binding(3) var<storage, read_write> C_f32: array<f32>;
25
+
26
+ var<workgroup> partials: array<f32, MAX_WORKGROUP_SIZE>;
27
+
28
+ fn packed_byte(byte_index: u32) -> u32 {
29
+ let word = B_words[byte_index >> 2u];
30
+ let shift = (byte_index & 3u) * 8u;
31
+ return (word >> shift) & 0xffu;
32
+ }
33
+
34
+ fn signed_int4(byte_value: u32, high_nibble: bool) -> f32 {
35
+ let value = select(byte_value & 0x0fu, (byte_value >> 4u) & 0x0fu, high_nibble);
36
+ if (STORAGE_OFFSET_BINARY == 1u) {
37
+ return f32(value) - 8.0;
38
+ }
39
+ return select(f32(value), f32(i32(value) - 16), value >= 8u);
40
+ }
41
+
42
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
43
+ fn main_multicol_f32a_f32out(
44
+ @builtin(local_invocation_id) lid: vec3<u32>,
45
+ @builtin(workgroup_id) wg_id: vec3<u32>
46
+ ) {
47
+ let local_id = lid.x;
48
+ let lanes_per_col = WORKGROUP_SIZE / MULTICOL_COLS_PER_WG;
49
+ let col_in_wg = local_id / lanes_per_col;
50
+ let lane_in_col = local_id - col_in_wg * lanes_per_col;
51
+ let row = wg_id.y;
52
+ let col = wg_id.x * MULTICOL_COLS_PER_WG + col_in_wg;
53
+ let packed_bytes_per_row = (u.K + 1u) >> 1u;
54
+
55
+ var sum: f32 = 0.0;
56
+ if (row < u.M && col < u.N) {
57
+ let row_base = col * packed_bytes_per_row;
58
+ for (var k: u32 = lane_in_col; k < u.K; k = k + lanes_per_col) {
59
+ let byte_value = packed_byte(row_base + (k >> 1u));
60
+ let weight = signed_int4(byte_value, (k & 1u) == 1u) * WEIGHT_SCALE;
61
+ sum = sum + A[row * u.K + k] * weight;
62
+ }
63
+ }
64
+
65
+ partials[local_id] = sum;
66
+ workgroupBarrier();
67
+
68
+ if (lane_in_col == 0u && row < u.M && col < u.N) {
69
+ var final_sum: f32 = 0.0;
70
+ let base = col_in_wg * lanes_per_col;
71
+ for (var lane: u32 = 0u; lane < lanes_per_col; lane = lane + 1u) {
72
+ final_sum = final_sum + partials[base + lane];
73
+ }
74
+ C_f32[row * u.N + col] = final_sum * u.alpha;
75
+ }
76
+ }
@@ -39,6 +39,16 @@ export function selectMatmulRMSNormFusedVariant(N, dtype) {
39
39
  }
40
40
 
41
41
 
42
+ function assertFusedOptions(options) {
43
+ if (options?.transposeB === undefined) {
44
+ throw new Error('[MatmulRMSNormFused] transposeB is required.');
45
+ }
46
+ if (options?.rmsNormWeightOffset === undefined) {
47
+ throw new Error('[MatmulRMSNormFused] rmsNormWeightOffset is required.');
48
+ }
49
+ }
50
+
51
+
42
52
  export async function runMatmulRMSNormFused(
43
53
  input,
44
54
  weight,
@@ -46,14 +56,15 @@ export async function runMatmulRMSNormFused(
46
56
  options
47
57
  ) {
48
58
  const device = getDevice();
59
+ assertFusedOptions(options);
49
60
  const {
50
61
  N,
51
62
  K,
52
63
  eps,
53
64
  residual = null,
54
65
  outputBuffer = null,
55
- transposeB = true, // Default: GGUF row-major weights
56
- rmsNormWeightOffset = false,
66
+ transposeB,
67
+ rmsNormWeightOffset,
57
68
  label = null,
58
69
  } = options;
59
70
  if (eps == null) {
@@ -91,34 +102,39 @@ export async function runMatmulRMSNormFused(
91
102
  // Output buffer: [1, N] - size depends on dtype
92
103
  const bytesPerElement = dtype === 'f16' ? 2 : 4;
93
104
  const outputSize = N * bytesPerElement;
94
- const ownedOutput = outputBuffer ? null : acquireBuffer(outputSize, undefined, 'matmul_rmsnorm_fused_output');
95
- const output = outputBuffer || ownedOutput;
96
-
97
- // Create uniform buffer (8 u32/f32 = 32 bytes, padded for alignment)
98
- const uniformBuffer = createUniformBufferWithView(
99
- 'matmul_rmsnorm_fused_uniforms',
100
- 32,
101
- (view) => {
102
- view.setUint32(0, N, true);
103
- view.setUint32(4, K, true);
104
- view.setFloat32(8, eps, true);
105
- view.setUint32(12, residual ? 1 : 0, true);
106
- view.setUint32(16, transposeB ? 1 : 0, true);
107
- // Padding bytes 20-31 are zero-initialized
108
- },
109
- null,
110
- device
111
- );
112
-
113
- // Create placeholder for residual if not provided
105
+ let ownedOutput = null;
106
+ let output = outputBuffer;
107
+ let uniformBuffer = null;
108
+ let residualBuffer = residual;
114
109
  const ownsResidualBuffer = !residual;
115
- const residualBuffer = residual || device.createBuffer({
116
- label: 'matmul_rmsnorm_residual_placeholder',
117
- size: 4,
118
- usage: GPUBufferUsage.STORAGE,
119
- });
120
-
110
+ let completed = false;
121
111
  try {
112
+ ownedOutput = outputBuffer ? null : acquireBuffer(outputSize, undefined, 'matmul_rmsnorm_fused_output');
113
+ output = outputBuffer || ownedOutput;
114
+
115
+ // Create uniform buffer (8 u32/f32 = 32 bytes, padded for alignment)
116
+ uniformBuffer = createUniformBufferWithView(
117
+ 'matmul_rmsnorm_fused_uniforms',
118
+ 32,
119
+ (view) => {
120
+ view.setUint32(0, N, true);
121
+ view.setUint32(4, K, true);
122
+ view.setFloat32(8, eps, true);
123
+ view.setUint32(12, residual ? 1 : 0, true);
124
+ view.setUint32(16, transposeB ? 1 : 0, true);
125
+ // Padding bytes 20-31 are zero-initialized
126
+ },
127
+ null,
128
+ device
129
+ );
130
+
131
+ // Create placeholder for residual if not provided
132
+ residualBuffer = residual || device.createBuffer({
133
+ label: 'matmul_rmsnorm_residual_placeholder',
134
+ size: 4,
135
+ usage: GPUBufferUsage.STORAGE,
136
+ });
137
+
122
138
  const bindGroup = device.createBindGroup({
123
139
  label: 'matmul_rmsnorm_fused_bind_group',
124
140
  layout: pipeline.getBindGroupLayout(0),
@@ -135,23 +151,20 @@ export async function runMatmulRMSNormFused(
135
151
  const workgroups = 1;
136
152
  const dispatchLabel = label ? `matmul_rmsnorm_fused:${label}` : 'matmul_rmsnorm_fused';
137
153
  dispatch(device, pipeline, bindGroup, workgroups, dispatchLabel);
138
- } catch (error) {
139
- uniformBuffer.destroy();
140
- if (ownsResidualBuffer) {
154
+
155
+ // Output dtype matches input dtype
156
+ const tensor = createTensor(output, input.dtype, [1, N], 'matmul_rmsnorm_fused_output');
157
+ completed = true;
158
+ return tensor;
159
+ } finally {
160
+ uniformBuffer?.destroy();
161
+ if (ownsResidualBuffer && residualBuffer) {
141
162
  residualBuffer.destroy();
142
163
  }
143
- if (ownedOutput) {
164
+ if (!completed && ownedOutput) {
144
165
  releaseBuffer(ownedOutput);
145
166
  }
146
- throw error;
147
167
  }
148
-
149
- // Cleanup
150
- uniformBuffer.destroy();
151
- if (ownsResidualBuffer) residualBuffer.destroy();
152
-
153
- // Output dtype matches input dtype
154
- return createTensor(output, input.dtype, [1, N], 'matmul_rmsnorm_fused_output');
155
168
  }
156
169
 
157
170
 
@@ -163,14 +176,15 @@ export async function recordMatmulRMSNormFused(
163
176
  options
164
177
  ) {
165
178
  const device = recorder.device;
179
+ assertFusedOptions(options);
166
180
  const {
167
181
  N,
168
182
  K,
169
183
  eps,
170
184
  residual = null,
171
185
  outputBuffer = null,
172
- transposeB = true, // Default: GGUF row-major weights
173
- rmsNormWeightOffset = false,
186
+ transposeB,
187
+ rmsNormWeightOffset,
174
188
  label = null,
175
189
  } = options;
176
190
  if (eps == null) {
@@ -208,33 +222,37 @@ export async function recordMatmulRMSNormFused(
208
222
  // Output buffer - size depends on dtype
209
223
  const bytesPerElement = dtype === 'f16' ? 2 : 4;
210
224
  const outputSize = N * bytesPerElement;
211
- const ownedOutput = outputBuffer ? null : acquireBuffer(outputSize, undefined, 'matmul_rmsnorm_fused_output');
212
- const output = outputBuffer || ownedOutput;
213
-
214
- // Uniform buffer via recorder (8 u32/f32 = 32 bytes, padded for alignment)
215
- const uniformBuffer = createUniformBufferWithView(
216
- 'matmul_rmsnorm_fused_uniforms',
217
- 32,
218
- (view) => {
219
- view.setUint32(0, N, true);
220
- view.setUint32(4, K, true);
221
- view.setFloat32(8, eps, true);
222
- view.setUint32(12, residual ? 1 : 0, true);
223
- view.setUint32(16, transposeB ? 1 : 0, true);
224
- // Padding bytes 20-31 are zero-initialized
225
- },
226
- recorder
227
- );
228
-
229
- // Placeholder for residual
225
+ let ownedOutput = null;
226
+ let output = outputBuffer;
227
+ let residualBuffer = residual;
230
228
  const ownsResidualBuffer = !residual;
231
- const residualBuffer = residual || device.createBuffer({
232
- label: 'matmul_rmsnorm_residual_placeholder',
233
- size: 4,
234
- usage: GPUBufferUsage.STORAGE,
235
- });
236
-
229
+ let completed = false;
237
230
  try {
231
+ ownedOutput = outputBuffer ? null : acquireBuffer(outputSize, undefined, 'matmul_rmsnorm_fused_output');
232
+ output = outputBuffer || ownedOutput;
233
+
234
+ // Uniform buffer via recorder (8 u32/f32 = 32 bytes, padded for alignment)
235
+ const uniformBuffer = createUniformBufferWithView(
236
+ 'matmul_rmsnorm_fused_uniforms',
237
+ 32,
238
+ (view) => {
239
+ view.setUint32(0, N, true);
240
+ view.setUint32(4, K, true);
241
+ view.setFloat32(8, eps, true);
242
+ view.setUint32(12, residual ? 1 : 0, true);
243
+ view.setUint32(16, transposeB ? 1 : 0, true);
244
+ // Padding bytes 20-31 are zero-initialized
245
+ },
246
+ recorder
247
+ );
248
+
249
+ // Placeholder for residual
250
+ residualBuffer = residual || device.createBuffer({
251
+ label: 'matmul_rmsnorm_residual_placeholder',
252
+ size: 4,
253
+ usage: GPUBufferUsage.STORAGE,
254
+ });
255
+
238
256
  const bindGroup = device.createBindGroup({
239
257
  label: 'matmul_rmsnorm_fused_bind_group',
240
258
  layout: pipeline.getBindGroupLayout(0),
@@ -251,23 +269,26 @@ export async function recordMatmulRMSNormFused(
251
269
  const workgroups = 1;
252
270
  const dispatchLabel = label ? `matmul_rmsnorm_fused:${label}` : 'matmul_rmsnorm_fused';
253
271
  recordDispatch(recorder, pipeline, bindGroup, workgroups, dispatchLabel);
254
- } catch (error) {
272
+
273
+ // Track placeholder for cleanup
255
274
  if (ownsResidualBuffer) {
256
- residualBuffer.destroy();
257
- }
258
- if (ownedOutput) {
259
- releaseBuffer(ownedOutput);
275
+ recorder.trackTemporaryBuffer(residualBuffer);
260
276
  }
261
- throw error;
262
- }
263
277
 
264
- // Track placeholder for cleanup
265
- if (ownsResidualBuffer) {
266
- recorder.trackTemporaryBuffer(residualBuffer);
278
+ // Output dtype matches input dtype
279
+ const tensor = createTensor(output, input.dtype, [1, N], 'matmul_rmsnorm_fused_output');
280
+ completed = true;
281
+ return tensor;
282
+ } finally {
283
+ if (!completed) {
284
+ if (ownsResidualBuffer && residualBuffer) {
285
+ residualBuffer.destroy();
286
+ }
287
+ if (ownedOutput) {
288
+ releaseBuffer(ownedOutput);
289
+ }
290
+ }
267
291
  }
268
-
269
- // Output dtype matches input dtype
270
- return createTensor(output, input.dtype, [1, N], 'matmul_rmsnorm_fused_output');
271
292
  }
272
293
 
273
294