doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -0,0 +1,128 @@
1
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
2
+ import { padToQ4KBlock } from '../../config/schema/index.js';
3
+ import { createTensor, dtypeBytes } from '../tensor.js';
4
+ import { getBuffer } from '../weight-buffer.js';
5
+ import { unifiedKernelWrapper } from './utils.js';
6
+ import { selectRuleValue } from './rule-registry.js';
7
+ import {
8
+ assertRMSNormWeightBuffer,
9
+ planRMSNormDispatch,
10
+ resolveNormWeightDtype,
11
+ } from './rmsnorm.js';
12
+
13
+ function assertPositiveInteger(value, label) {
14
+ if (!Number.isInteger(value) || value <= 0) {
15
+ throw new Error(`RMSNorm QK requires ${label} to be a positive integer.`);
16
+ }
17
+ }
18
+
19
+ function assertBufferCapacity(buffer, requiredBytes, label) {
20
+ if (Number.isFinite(buffer?.size) && requiredBytes > buffer.size) {
21
+ throw new Error(`RMSNorm QK ${label} buffer is smaller than requested range (${requiredBytes} > ${buffer.size} bytes).`);
22
+ }
23
+ }
24
+
25
+ function selectRMSNormQKVariant(dtype) {
26
+ return selectRuleValue('rmsnormQk', 'variant', { isF16: dtype === 'f16' });
27
+ }
28
+
29
+ export function canUseRMSNormQK(qTensor, kTensor, options = {}) {
30
+ if (options.skipKNorm === true) {
31
+ return false;
32
+ }
33
+ if (!qTensor || !kTensor || qTensor.buffer === kTensor.buffer) {
34
+ return false;
35
+ }
36
+ return qTensor.dtype === kTensor.dtype && (qTensor.dtype === 'f32' || qTensor.dtype === 'f16');
37
+ }
38
+
39
+ async function rmsNormQK(target, qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
40
+ const {
41
+ numTokens,
42
+ numHeads,
43
+ numKVHeads,
44
+ headDim,
45
+ rmsNormWeightOffset = false,
46
+ } = options;
47
+
48
+ assertPositiveInteger(numTokens, 'numTokens');
49
+ assertPositiveInteger(numHeads, 'numHeads');
50
+ assertPositiveInteger(numKVHeads, 'numKVHeads');
51
+ assertPositiveInteger(headDim, 'headDim');
52
+ if (qTensor.dtype !== kTensor.dtype) {
53
+ throw new Error(`RMSNorm QK requires matching Q/K dtypes, got ${qTensor.dtype} and ${kTensor.dtype}.`);
54
+ }
55
+ if (qTensor.dtype !== 'f32' && qTensor.dtype !== 'f16') {
56
+ throw new Error(`RMSNorm QK requires f32 or f16 tensors, got ${qTensor.dtype}.`);
57
+ }
58
+
59
+ const qRows = numTokens * numHeads;
60
+ const kRows = numTokens * numKVHeads;
61
+ const bytesPerElement = dtypeBytes(qTensor.dtype);
62
+ assertBufferCapacity(qTensor.buffer, qRows * headDim * bytesPerElement, 'Q input');
63
+ assertBufferCapacity(kTensor.buffer, kRows * headDim * bytesPerElement, 'K input');
64
+
65
+ const qWeightBuffer = getBuffer(qWeight);
66
+ const kWeightBuffer = getBuffer(kWeight);
67
+ assertRMSNormWeightBuffer(qWeight, qWeightBuffer, headDim);
68
+ assertRMSNormWeightBuffer(kWeight, kWeightBuffer, headDim);
69
+ const qWeightDtype = resolveNormWeightDtype(qWeight, headDim);
70
+ const kWeightDtype = resolveNormWeightDtype(kWeight, headDim);
71
+
72
+ const paddedHeadDim = padToQ4KBlock(headDim);
73
+ const qOutputSize = qRows * paddedHeadDim * bytesPerElement;
74
+ const kOutputSize = kRows * paddedHeadDim * bytesPerElement;
75
+ let qOutputBuffer = null;
76
+ let kOutputBuffer = null;
77
+
78
+ try {
79
+ qOutputBuffer = acquireBuffer(qOutputSize, undefined, 'rmsnorm_qk_q_output');
80
+ kOutputBuffer = acquireBuffer(kOutputSize, undefined, 'rmsnorm_qk_k_output');
81
+ const dispatchPlan = planRMSNormDispatch(target, qRows + kRows);
82
+ await unifiedKernelWrapper(
83
+ 'rmsnorm_qk',
84
+ target,
85
+ selectRMSNormQKVariant(qTensor.dtype),
86
+ [qTensor, qWeightBuffer, qOutputBuffer, kTensor, kWeightBuffer, kOutputBuffer],
87
+ {
88
+ q_rows: qRows,
89
+ k_rows: kRows,
90
+ head_dim: headDim,
91
+ row_stride: dispatchPlan.tokenStride,
92
+ eps,
93
+ _pad0: 0,
94
+ _pad1: 0,
95
+ _pad2: 0,
96
+ },
97
+ dispatchPlan.workgroups,
98
+ {
99
+ RMS_NORM_OFFSET: rmsNormWeightOffset,
100
+ Q_WEIGHT_IS_F16: qWeightDtype === 'f16',
101
+ K_WEIGHT_IS_F16: kWeightDtype === 'f16',
102
+ },
103
+ null,
104
+ 'rmsnorm_qk'
105
+ );
106
+
107
+ return {
108
+ q: createTensor(qOutputBuffer, qTensor.dtype, [qRows, headDim], 'rmsnorm_qk_q_output'),
109
+ k: createTensor(kOutputBuffer, kTensor.dtype, [kRows, headDim], 'rmsnorm_qk_k_output'),
110
+ };
111
+ } catch (error) {
112
+ if (qOutputBuffer) {
113
+ releaseBuffer(qOutputBuffer);
114
+ }
115
+ if (kOutputBuffer) {
116
+ releaseBuffer(kOutputBuffer);
117
+ }
118
+ throw error;
119
+ }
120
+ }
121
+
122
+ export async function runRMSNormQK(qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
123
+ return rmsNormQK(null, qTensor, kTensor, qWeight, kWeight, eps, options);
124
+ }
125
+
126
+ export async function recordRMSNormQK(recorder, qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
127
+ return rmsNormQK(recorder, qTensor, kTensor, qWeight, kWeight, eps, options);
128
+ }
@@ -9,15 +9,46 @@ import type { CommandRecorder } from '../command-recorder.js';
9
9
  import type { OutputBufferOptions } from './types.js';
10
10
  import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
11
11
 
12
+ /** Maximum hidden size supported by the cached residual RMSNorm variant. */
13
+ export declare const RMSNORM_CACHE_LIMIT: number;
14
+
12
15
  /** RMSNorm kernel options */
13
16
  export interface RMSNormOptions extends OutputBufferOptions {
14
17
  batchSize?: number;
15
18
  hiddenSize?: number | null;
16
19
  residual?: Tensor | null;
20
+ preResidual?: Tensor | null;
21
+ residualSumOutput?: GPUBuffer | Tensor | null;
22
+ outputScale?: number | null;
17
23
  /** Use (1+w)*x normalization for Gemma 2/3 */
18
24
  rmsNormWeightOffset?: boolean;
25
+ label?: string;
19
26
  }
20
27
 
28
+ /**
29
+ * Return true when the residual RMSNorm variant must avoid the cached WGSL path.
30
+ */
31
+ export declare function residualVariantBypassesCache(
32
+ residual: Tensor | GPUBuffer | WeightBuffer | TensorLike | null | undefined,
33
+ hiddenSize: number | null | undefined
34
+ ): boolean;
35
+
36
+ export declare function resolveNormWeightDtype(
37
+ weight: GPUBuffer | WeightBuffer | TensorLike,
38
+ hiddenSize: number | null | undefined
39
+ ): string;
40
+
41
+ export declare function assertRMSNormWeightBuffer(
42
+ weight: GPUBuffer | WeightBuffer | TensorLike,
43
+ weightBuffer: GPUBuffer | null | undefined,
44
+ hiddenSize: number | null | undefined
45
+ ): void;
46
+
47
+ export declare function planRMSNormDispatch(
48
+ target: CommandRecorder | null | undefined,
49
+ numTokens: number
50
+ ): { tokenStride: number; workgroups: [number, number, number] };
51
+
21
52
  /**
22
53
  * Select RMSNorm kernel variant based on options, tensor dtypes, and GPU capabilities.
23
54
  */
@@ -41,7 +41,7 @@ function normalizeNormWeightDtype(dtype) {
41
41
  return null;
42
42
  }
43
43
 
44
- function resolveNormWeightDtype(weight, hiddenSize) {
44
+ export function resolveNormWeightDtype(weight, hiddenSize) {
45
45
  const explicitDtype = normalizeNormWeightDtype(getWeightDtype(weight));
46
46
  if (explicitDtype) {
47
47
  return explicitDtype;
@@ -76,7 +76,7 @@ function resolveNormWeightDtype(weight, hiddenSize) {
76
76
  return DEFAULT_DTYPE;
77
77
  }
78
78
 
79
- function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
79
+ export function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
80
80
  const isGpuBuffer = weightBuffer && (
81
81
  typeof GPUBuffer === 'undefined'
82
82
  ? true
@@ -94,7 +94,7 @@ function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
94
94
  );
95
95
  }
96
96
 
97
- function planRMSNormDispatch(target, numTokens) {
97
+ export function planRMSNormDispatch(target, numTokens) {
98
98
  const device = target?.device;
99
99
  const maxPerDim = Number.isFinite(device?.limits?.maxComputeWorkgroupsPerDimension)
100
100
  ? device.limits.maxComputeWorkgroupsPerDimension
@@ -106,6 +106,25 @@ function planRMSNormDispatch(target, numTokens) {
106
106
  };
107
107
  }
108
108
 
109
+ function resolveRMSNormOutputScale(outputScale) {
110
+ if (outputScale == null) {
111
+ return 1;
112
+ }
113
+ const value = Number(outputScale);
114
+ if (!Number.isFinite(value)) {
115
+ throw new Error(`[rmsnorm] outputScale must be finite; got "${String(outputScale)}".`);
116
+ }
117
+ return value;
118
+ }
119
+
120
+ function resolveRMSNormDispatchLabel(label) {
121
+ if (typeof label !== 'string' || label.length === 0) {
122
+ return 'rmsnorm';
123
+ }
124
+ const normalized = label.replace(/^L\d+\./, '').replace(/\s+/g, '_');
125
+ return `rmsnorm:${normalized}`;
126
+ }
127
+
109
128
  export function selectRMSNormKernel(options = {}, isF16 = false) {
110
129
  const { residual = null, hiddenSize = null } = options;
111
130
  const { smallThreshold } = getKernelThresholds().rmsnorm;
@@ -133,7 +152,9 @@ export async function runRMSNorm(
133
152
  const {
134
153
  batchSize = 1, hiddenSize, residual = null, outputBuffer = null,
135
154
  rmsNormWeightOffset = false, preResidual = null, residualSumOutput = null,
155
+ outputScale = null,
136
156
  } = options;
157
+ const resolvedOutputScale = resolveRMSNormOutputScale(outputScale);
137
158
  const isF16 = input.dtype === 'f16';
138
159
  const variant = selectRMSNormKernel(options, isF16);
139
160
  const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
@@ -173,7 +194,7 @@ export async function runRMSNorm(
173
194
  eps,
174
195
  has_residual: residual ? 1 : 0,
175
196
  token_stride: dispatchPlan.tokenStride,
176
- _pad0: 0,
197
+ output_scale: resolvedOutputScale,
177
198
  _pad1: 0,
178
199
  _pad2: 0,
179
200
  },
@@ -184,7 +205,8 @@ export async function runRMSNorm(
184
205
  PRE_RESIDUAL: !!preResidual,
185
206
  OUTPUT_PRENORM: hasPrenormOutput,
186
207
  },
187
- extraBindings
208
+ extraBindings,
209
+ resolveRMSNormDispatchLabel(options.label)
188
210
  );
189
211
 
190
212
  if (ownedPrenormPlaceholder) releaseBuffer(ownedPrenormPlaceholder);
@@ -209,7 +231,9 @@ export async function recordRMSNorm(
209
231
  const {
210
232
  batchSize = 1, hiddenSize = null, residual = null, outputBuffer = null,
211
233
  rmsNormWeightOffset = false, preResidual = null, residualSumOutput = null,
234
+ outputScale = null,
212
235
  } = options;
236
+ const resolvedOutputScale = resolveRMSNormOutputScale(outputScale);
213
237
  const isF16 = input.dtype === 'f16';
214
238
  const variant = selectRMSNormKernel(options, isF16);
215
239
  const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
@@ -246,7 +270,7 @@ export async function recordRMSNorm(
246
270
  eps,
247
271
  has_residual: residual ? 1 : 0,
248
272
  token_stride: dispatchPlan.tokenStride,
249
- _pad0: 0,
273
+ output_scale: resolvedOutputScale,
250
274
  _pad1: 0,
251
275
  _pad2: 0,
252
276
  },
@@ -257,7 +281,8 @@ export async function recordRMSNorm(
257
281
  PRE_RESIDUAL: !!preResidual,
258
282
  OUTPUT_PRENORM: hasPrenormOutput,
259
283
  },
260
- extraBindings
284
+ extraBindings,
285
+ resolveRMSNormDispatchLabel(options.label)
261
286
  );
262
287
 
263
288
  if (ownedPrenormPlaceholder) releaseBuffer(ownedPrenormPlaceholder);
@@ -42,7 +42,7 @@ struct Uniforms {
42
42
  eps: f32, // Epsilon for numerical stability (typically 1e-5 or 1e-6)
43
43
  has_residual: u32, // Runtime flag: 1 = add residual after norm
44
44
  token_stride: u32, // Workgroup rows per dispatch row
45
- _pad0: u32,
45
+ output_scale: f32, // Output epilogue scale
46
46
  _pad1: u32,
47
47
  _pad2: u32,
48
48
  }
@@ -103,6 +103,10 @@ fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
103
103
  }
104
104
  }
105
105
 
106
+ fn apply_output_scale(value: f32) -> f32 {
107
+ return value * u.output_scale;
108
+ }
109
+
106
110
  fn token_index(wg_id: vec3<u32>) -> u32 {
107
111
  return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
108
112
  }
@@ -177,7 +181,7 @@ fn main(
177
181
  // Write pre-norm sum for downstream residual reuse
178
182
  write_prenorm(base_offset, idx, x);
179
183
 
180
- output[base_offset + idx] = result;
184
+ output[base_offset + idx] = apply_output_scale(result);
181
185
  }
182
186
  }
183
187
  }
@@ -231,7 +235,7 @@ fn main_small(
231
235
  result = result + residual[base_offset + thread_idx];
232
236
  }
233
237
  write_prenorm(base_offset, thread_idx, x);
234
- output[base_offset + thread_idx] = result;
238
+ output[base_offset + thread_idx] = apply_output_scale(result);
235
239
  }
236
240
  }
237
241
 
@@ -295,7 +299,7 @@ fn main_cached(
295
299
  result = result + residual[base_offset + idx];
296
300
  }
297
301
  write_prenorm(base_offset, idx, x);
298
- output[base_offset + idx] = result;
302
+ output[base_offset + idx] = apply_output_scale(result);
299
303
  }
300
304
  }
301
305
  }
@@ -378,7 +382,7 @@ fn main_subgroup(
378
382
  }
379
383
 
380
384
  write_prenorm(base_offset, idx, x);
381
- output[base_offset + idx] = result;
385
+ output[base_offset + idx] = apply_output_scale(result);
382
386
  }
383
387
  }
384
388
  }
@@ -443,6 +447,6 @@ fn main_small_subgroup(
443
447
  result = result + residual[base_offset + thread_idx];
444
448
  }
445
449
  write_prenorm(base_offset, thread_idx, x);
446
- output[base_offset + thread_idx] = result;
450
+ output[base_offset + thread_idx] = apply_output_scale(result);
447
451
  }
448
452
  }
@@ -1,5 +1,9 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
2
2
  // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
+ // AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
4
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
5
+ // AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
6
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
7
  //
4
8
  // F16 variant for reduced memory bandwidth when using F16 activations.
5
9
  // The all-f16 lane keeps normalization arithmetic in f16.
@@ -22,7 +26,7 @@ struct Uniforms {
22
26
  eps: f32, // Epsilon for numerical stability
23
27
  has_residual: u32, // 1 if residual input provided, 0 otherwise
24
28
  token_stride: u32, // Workgroup rows per dispatch row
25
- _pad0: u32,
29
+ output_scale: f32, // Output epilogue scale
26
30
  _pad1: u32,
27
31
  _pad2: u32,
28
32
  }
@@ -156,7 +160,7 @@ fn main(
156
160
  // Write pre-norm sum for downstream residual reuse
157
161
  write_prenorm(base_offset, idx, x);
158
162
 
159
- output[base_offset + idx] = result;
163
+ output[base_offset + idx] = f16(f32(result) * u.output_scale);
160
164
  }
161
165
  }
162
166
  }
@@ -219,6 +223,6 @@ fn rmsnorm_small_f16(
219
223
  result = result + residual[base_offset + thread_idx];
220
224
  }
221
225
  write_prenorm(base_offset, thread_idx, x);
222
- output[base_offset + thread_idx] = result;
226
+ output[base_offset + thread_idx] = f16(f32(result) * u.output_scale);
223
227
  }
224
228
  }
@@ -0,0 +1,224 @@
1
+ // Sandwich RMSNorm pair kernel.
2
+ //
3
+ // Computes:
4
+ // post_output = residual + rmsnorm(input, post_weight)
5
+ // pre_output = rmsnorm(post_output, pre_weight)
6
+ //
7
+ // This is intended for decode sandwich-norm paths where post_attn_norm is
8
+ // immediately followed by pre_ffn_norm and both tensors are needed downstream.
9
+
10
+ enable subgroups;
11
+
12
+ override WORKGROUP_SIZE: u32 = 256u;
13
+ override RMS_NORM_OFFSET: bool = false;
14
+ override POST_WEIGHT_IS_F16: bool = false;
15
+ override PRE_WEIGHT_IS_F16: bool = false;
16
+
17
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
18
+ const MAX_SUBGROUPS: u32 = 32u;
19
+ const MAX_PAIR_CACHE_SIZE: u32 = 4608u;
20
+
21
+ struct Uniforms {
22
+ size: u32,
23
+ num_tokens: u32,
24
+ eps: f32,
25
+ has_residual: u32,
26
+ token_stride: u32,
27
+ _pad0: u32,
28
+ _pad1: u32,
29
+ _pad2: u32,
30
+ }
31
+
32
+ @group(0) @binding(0) var<uniform> u: Uniforms;
33
+ @group(0) @binding(1) var<storage, read> input: array<f32>;
34
+ @group(0) @binding(2) var<storage, read> residual: array<f32>;
35
+ @group(0) @binding(3) var<storage, read> post_weight: array<u32>;
36
+ @group(0) @binding(4) var<storage, read> pre_weight: array<u32>;
37
+ @group(0) @binding(5) var<storage, read_write> post_output: array<f32>;
38
+ @group(0) @binding(6) var<storage, read_write> pre_output: array<f32>;
39
+
40
+ var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
41
+ var<workgroup> shared_post: array<f32, MAX_PAIR_CACHE_SIZE>;
42
+ var<workgroup> sg_partial_sums: array<f32, MAX_SUBGROUPS>;
43
+
44
+ fn apply_weight(w: f32) -> f32 {
45
+ if (RMS_NORM_OFFSET) {
46
+ return 1.0 + w;
47
+ }
48
+ return w;
49
+ }
50
+
51
+ fn load_post_weight(idx: u32) -> f32 {
52
+ if (POST_WEIGHT_IS_F16) {
53
+ let packed = post_weight[idx >> 1u];
54
+ let pair = unpack2x16float(packed);
55
+ return select(pair.x, pair.y, (idx & 1u) == 1u);
56
+ }
57
+ return bitcast<f32>(post_weight[idx]);
58
+ }
59
+
60
+ fn load_pre_weight(idx: u32) -> f32 {
61
+ if (PRE_WEIGHT_IS_F16) {
62
+ let packed = pre_weight[idx >> 1u];
63
+ let pair = unpack2x16float(packed);
64
+ return select(pair.x, pair.y, (idx & 1u) == 1u);
65
+ }
66
+ return bitcast<f32>(pre_weight[idx]);
67
+ }
68
+
69
+ fn token_index(wg_id: vec3<u32>) -> u32 {
70
+ return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
71
+ }
72
+
73
+ fn reduce_workgroup_sum(local_sum: f32, thread_idx: u32) -> f32 {
74
+ shared_sum[thread_idx] = local_sum;
75
+ workgroupBarrier();
76
+
77
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
78
+ if (thread_idx < stride) {
79
+ shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
80
+ }
81
+ workgroupBarrier();
82
+ }
83
+
84
+ return shared_sum[0];
85
+ }
86
+
87
+ fn reduce_subgroup_sum(local_sum: f32, thread_idx: u32, sg_lane: u32, sg_size: u32) -> f32 {
88
+ let subgroup_id = thread_idx / sg_size;
89
+ let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
90
+ let sg_sum = subgroupAdd(local_sum);
91
+
92
+ if (sg_lane == 0u && subgroup_id < num_subgroups) {
93
+ sg_partial_sums[subgroup_id] = sg_sum;
94
+ }
95
+ workgroupBarrier();
96
+
97
+ if (thread_idx == 0u) {
98
+ var sum: f32 = 0.0;
99
+ for (var s = 0u; s < num_subgroups; s = s + 1u) {
100
+ sum = sum + sg_partial_sums[s];
101
+ }
102
+ sg_partial_sums[0] = sum;
103
+ }
104
+ workgroupBarrier();
105
+
106
+ return sg_partial_sums[0];
107
+ }
108
+
109
+ fn compute_post(base_offset: u32, idx: u32, input_inv_rms: f32) -> f32 {
110
+ let source = input[base_offset + idx];
111
+ var post = source * input_inv_rms * apply_weight(load_post_weight(idx));
112
+ if (u.has_residual != 0u) {
113
+ post = post + residual[base_offset + idx];
114
+ }
115
+ return post;
116
+ }
117
+
118
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
119
+ fn main(
120
+ @builtin(local_invocation_id) local_id: vec3<u32>,
121
+ @builtin(workgroup_id) wg_id: vec3<u32>
122
+ ) {
123
+ let token_idx = token_index(wg_id);
124
+ let thread_idx = local_id.x;
125
+ let size = u.size;
126
+
127
+ if (token_idx >= u.num_tokens) {
128
+ return;
129
+ }
130
+
131
+ let base_offset = token_idx * size;
132
+ let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
133
+
134
+ var input_sum_sq: f32 = 0.0;
135
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
136
+ let idx = thread_idx * elements_per_thread + i;
137
+ if (idx < size) {
138
+ let source = input[base_offset + idx];
139
+ input_sum_sq = input_sum_sq + source * source;
140
+ }
141
+ }
142
+
143
+ let input_total = reduce_workgroup_sum(input_sum_sq, thread_idx);
144
+ let input_inv_rms = 1.0 / sqrt(input_total / f32(size) + u.eps);
145
+ workgroupBarrier();
146
+
147
+ var post_sum_sq: f32 = 0.0;
148
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
149
+ let idx = thread_idx * elements_per_thread + i;
150
+ if (idx < size) {
151
+ let post = compute_post(base_offset, idx, input_inv_rms);
152
+ shared_post[idx] = post;
153
+ post_output[base_offset + idx] = post;
154
+ post_sum_sq = post_sum_sq + post * post;
155
+ }
156
+ }
157
+
158
+ let post_total = reduce_workgroup_sum(post_sum_sq, thread_idx);
159
+ let post_inv_rms = 1.0 / sqrt(post_total / f32(size) + u.eps);
160
+ workgroupBarrier();
161
+
162
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
163
+ let idx = thread_idx * elements_per_thread + i;
164
+ if (idx < size) {
165
+ let post = shared_post[idx];
166
+ pre_output[base_offset + idx] = post * post_inv_rms * apply_weight(load_pre_weight(idx));
167
+ }
168
+ }
169
+ }
170
+
171
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
172
+ fn main_subgroup(
173
+ @builtin(local_invocation_id) local_id: vec3<u32>,
174
+ @builtin(workgroup_id) wg_id: vec3<u32>,
175
+ @builtin(subgroup_invocation_id) sg_lane: u32,
176
+ @builtin(subgroup_size) sg_size: u32,
177
+ ) {
178
+ let token_idx = token_index(wg_id);
179
+ let thread_idx = local_id.x;
180
+ let size = u.size;
181
+
182
+ if (token_idx >= u.num_tokens) {
183
+ return;
184
+ }
185
+
186
+ let base_offset = token_idx * size;
187
+ let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
188
+
189
+ var input_sum_sq: f32 = 0.0;
190
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
191
+ let idx = thread_idx * elements_per_thread + i;
192
+ if (idx < size) {
193
+ let source = input[base_offset + idx];
194
+ input_sum_sq = input_sum_sq + source * source;
195
+ }
196
+ }
197
+
198
+ let input_total = reduce_subgroup_sum(input_sum_sq, thread_idx, sg_lane, sg_size);
199
+ let input_inv_rms = 1.0 / sqrt(input_total / f32(size) + u.eps);
200
+ workgroupBarrier();
201
+
202
+ var post_sum_sq: f32 = 0.0;
203
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
204
+ let idx = thread_idx * elements_per_thread + i;
205
+ if (idx < size) {
206
+ let post = compute_post(base_offset, idx, input_inv_rms);
207
+ shared_post[idx] = post;
208
+ post_output[base_offset + idx] = post;
209
+ post_sum_sq = post_sum_sq + post * post;
210
+ }
211
+ }
212
+
213
+ let post_total = reduce_subgroup_sum(post_sum_sq, thread_idx, sg_lane, sg_size);
214
+ let post_inv_rms = 1.0 / sqrt(post_total / f32(size) + u.eps);
215
+ workgroupBarrier();
216
+
217
+ for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
218
+ let idx = thread_idx * elements_per_thread + i;
219
+ if (idx < size) {
220
+ let post = shared_post[idx];
221
+ pre_output[base_offset + idx] = post * post_inv_rms * apply_weight(load_pre_weight(idx));
222
+ }
223
+ }
224
+ }