doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -145,26 +145,46 @@ function getMoEGatherBindGroupLayout(device) {
145
145
  // Required because auto layout can omit bindings in some driver/compiler paths.
146
146
  let scatterAddDynamicBindGroupLayout = null;
147
147
  let scatterAddDynamicBindGroupLayoutEpoch = -1;
148
+ let scatterAddDynamicScaledBindGroupLayout = null;
149
+ let scatterAddDynamicScaledBindGroupLayoutEpoch = -1;
148
150
 
149
- function getScatterAddDynamicBindGroupLayout(device) {
151
+ function getScatterAddDynamicBindGroupLayout(device, hasExpertScale = false) {
150
152
  const epoch = getDeviceEpoch();
151
- if (scatterAddDynamicBindGroupLayout && scatterAddDynamicBindGroupLayoutEpoch === epoch) {
153
+ if (!hasExpertScale && scatterAddDynamicBindGroupLayout && scatterAddDynamicBindGroupLayoutEpoch === epoch) {
152
154
  return scatterAddDynamicBindGroupLayout;
153
155
  }
156
+ if (hasExpertScale && scatterAddDynamicScaledBindGroupLayout && scatterAddDynamicScaledBindGroupLayoutEpoch === epoch) {
157
+ return scatterAddDynamicScaledBindGroupLayout;
158
+ }
154
159
 
155
- scatterAddDynamicBindGroupLayout = device.createBindGroupLayout({
156
- label: 'scatter_add_dynamic_explicit_layout',
160
+ const layout = device.createBindGroupLayout({
161
+ label: hasExpertScale
162
+ ? 'scatter_add_dynamic_scaled_explicit_layout'
163
+ : 'scatter_add_dynamic_explicit_layout',
157
164
  entries: [
158
165
  { binding: 0, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'uniform' } },
159
166
  { binding: 1, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
160
167
  { binding: 2, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
161
168
  { binding: 3, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
162
169
  { binding: 4, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
163
- { binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
170
+ ...(hasExpertScale
171
+ ? [
172
+ { binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
173
+ { binding: 6, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
174
+ ]
175
+ : [
176
+ { binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
177
+ ]),
164
178
  ],
165
179
  });
166
- scatterAddDynamicBindGroupLayoutEpoch = epoch;
167
- return scatterAddDynamicBindGroupLayout;
180
+ if (hasExpertScale) {
181
+ scatterAddDynamicScaledBindGroupLayout = layout;
182
+ scatterAddDynamicScaledBindGroupLayoutEpoch = epoch;
183
+ } else {
184
+ scatterAddDynamicBindGroupLayout = layout;
185
+ scatterAddDynamicBindGroupLayoutEpoch = epoch;
186
+ }
187
+ return layout;
168
188
  }
169
189
 
170
190
  let moeOffsetsBindGroupLayout = null;
@@ -450,8 +470,9 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
450
470
 
451
471
  async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
452
472
  const execution = resolveExecution(recorder);
453
- const { outputBuffer = null, weightsDtype = 'f32' } = options;
473
+ const { outputBuffer = null, weightsDtype = 'f32', perExpertScale = null } = options;
454
474
  const ownsOutput = outputBuffer == null;
475
+ const hasExpertScale = perExpertScale != null;
455
476
 
456
477
  if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
457
478
  throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
@@ -460,8 +481,9 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
460
481
  const variant = selectKernelRuleValue('moe', 'scatterAddVariant', {
461
482
  outputDtype: expertOutputs.dtype,
462
483
  weightsDtype,
484
+ hasExpertScale,
463
485
  });
464
- const explicitLayout = getScatterAddDynamicBindGroupLayout(execution.device);
486
+ const explicitLayout = getScatterAddDynamicBindGroupLayout(execution.device, hasExpertScale);
465
487
  const pipeline = await createPipeline('scatter_add', variant, explicitLayout);
466
488
 
467
489
  const bytesPerElement = expertOutputs.dtype === 'f16' ? 2 : 4;
@@ -475,17 +497,24 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
475
497
  });
476
498
 
477
499
  try {
500
+ const entries = [
501
+ { binding: 0, resource: { buffer: uniformBuffer } },
502
+ { binding: 1, resource: { buffer: expertOutputs.buffer } },
503
+ { binding: 2, resource: { buffer: indices } },
504
+ { binding: 3, resource: { buffer: weights } },
505
+ { binding: 4, resource: { buffer: tokenOffsets } },
506
+ ];
507
+ if (hasExpertScale) {
508
+ entries.push({ binding: 5, resource: { buffer: perExpertScale } });
509
+ entries.push({ binding: 6, resource: { buffer: outputBuf } });
510
+ } else {
511
+ entries.push({ binding: 5, resource: { buffer: outputBuf } });
512
+ }
513
+
478
514
  const bindGroup = await createBindGroupWithValidation(execution.device, {
479
515
  label: 'scatter_add_dynamic_bind_group',
480
516
  layout: pipeline.getBindGroupLayout(0),
481
- entries: [
482
- { binding: 0, resource: { buffer: uniformBuffer } },
483
- { binding: 1, resource: { buffer: expertOutputs.buffer } },
484
- { binding: 2, resource: { buffer: indices } },
485
- { binding: 3, resource: { buffer: weights } },
486
- { binding: 4, resource: { buffer: tokenOffsets } },
487
- { binding: 5, resource: { buffer: outputBuf } },
488
- ],
517
+ entries,
489
518
  }, `scatter_add_dynamic:${variant}`);
490
519
 
491
520
  const workgroups = Math.ceil((numTokens * topK * hiddenSize) / WORKGROUP_SIZES.DEFAULT);
@@ -524,4 +553,3 @@ export async function runScatterAddDynamic(expertOutputs, indices, weights, toke
524
553
  return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
525
554
  }
526
555
 
527
-
@@ -33,6 +33,14 @@ export declare function getOrCreatePipelineLayout(
33
33
  deviceOverride?: GPUDevice | null
34
34
  ): GPUPipelineLayout;
35
35
 
36
+ /**
37
+ * Get a cached bind group layout from a compute pipeline.
38
+ */
39
+ export declare function getPipelineBindGroupLayout(
40
+ pipeline: GPUComputePipeline,
41
+ index?: number
42
+ ): GPUBindGroupLayout;
43
+
36
44
  // ============================================================================
37
45
  // Pipeline Creation
38
46
  // ============================================================================
@@ -19,6 +19,8 @@ const bindGroupLayoutCache = new Map();
19
19
 
20
20
  const pipelineLayoutCache = new Map();
21
21
 
22
+ let pipelineBindGroupLayoutCache = new WeakMap();
23
+
22
24
  let pipelineCacheEpoch = -1;
23
25
  const deviceIds = new WeakMap();
24
26
  let nextDeviceId = 1;
@@ -38,6 +40,7 @@ function ensureCacheEpoch() {
38
40
  pipelineCache.clear();
39
41
  bindGroupLayoutCache.clear();
40
42
  pipelineLayoutCache.clear();
43
+ pipelineBindGroupLayoutCache = new WeakMap();
41
44
  pipelineCacheEpoch = epoch;
42
45
  }
43
46
  }
@@ -100,6 +103,26 @@ export function getOrCreatePipelineLayout(
100
103
  return layout;
101
104
  }
102
105
 
106
+ export function getPipelineBindGroupLayout(pipeline, index = 0) {
107
+ ensureCacheEpoch();
108
+ if (!pipeline || typeof pipeline.getBindGroupLayout !== 'function') {
109
+ throw new Error('getPipelineBindGroupLayout requires a GPUComputePipeline');
110
+ }
111
+ const normalizedIndex = Number.isInteger(index) && index >= 0 ? index : 0;
112
+ let layoutsByIndex = pipelineBindGroupLayoutCache.get(pipeline);
113
+ if (!layoutsByIndex) {
114
+ layoutsByIndex = new Map();
115
+ pipelineBindGroupLayoutCache.set(pipeline, layoutsByIndex);
116
+ }
117
+ const cached = layoutsByIndex.get(normalizedIndex);
118
+ if (cached) {
119
+ return cached;
120
+ }
121
+ const layout = pipeline.getBindGroupLayout(normalizedIndex);
122
+ layoutsByIndex.set(normalizedIndex, layout);
123
+ return layout;
124
+ }
125
+
103
126
  // ============================================================================
104
127
  // Pipeline Creation
105
128
  // ============================================================================
@@ -292,6 +315,7 @@ export function clearPipelineCaches() {
292
315
  pipelineCache.clear();
293
316
  bindGroupLayoutCache.clear();
294
317
  pipelineLayoutCache.clear();
318
+ pipelineBindGroupLayoutCache = new WeakMap();
295
319
  pipelineCacheEpoch = getDeviceEpoch();
296
320
  }
297
321
 
@@ -0,0 +1,21 @@
1
+ export interface RepPenaltyOptions {
2
+ vocabSize: number;
3
+ historyCount: number;
4
+ penalty: number;
5
+ batchCount: number;
6
+ batchOffset: number;
7
+ logitsDtype: 'f16' | 'f32';
8
+ }
9
+
10
+ export declare function selectRepPenaltyVariant(useF16: boolean): 'default' | 'default_f16';
11
+
12
+ export declare function recordRepPenalty(
13
+ recorder: {
14
+ device: GPUDevice;
15
+ beginComputePass(label: string): GPUComputePassEncoder;
16
+ },
17
+ logitsBuffer: GPUBuffer,
18
+ historyBuffer: GPUBuffer,
19
+ batchTokensBuffer: GPUBuffer,
20
+ options: RepPenaltyOptions
21
+ ): Promise<void>;
@@ -1,10 +1,8 @@
1
-
2
-
3
-
4
1
  import { getDevice, getKernelCapabilities } from '../device.js';
5
2
  import { createPipeline, getOrCreateBindGroupLayout } from './pipeline-cache.js';
6
3
  import { createUniformBufferWithView } from './uniform-utils.js';
7
4
  import { WORKGROUP_SIZES } from './constants.js';
5
+ import { selectRuleValue } from './rule-registry.js';
8
6
 
9
7
 
10
8
  function getRepPenaltyBindGroupLayout(device) {
@@ -20,8 +18,8 @@ function getRepPenaltyBindGroupLayout(device) {
20
18
  );
21
19
  }
22
20
 
23
- function resolveVariant(useF16) {
24
- return useF16 ? 'default_f16' : 'default';
21
+ export function selectRepPenaltyVariant(useF16) {
22
+ return selectRuleValue('repPenalty', 'variant', { useF16 });
25
23
  }
26
24
 
27
25
  function createRepPenaltyUniformBuffer(device, recorder, options) {
@@ -54,11 +52,21 @@ export async function recordRepPenalty(
54
52
  vocabSize,
55
53
  historyCount,
56
54
  penalty,
57
- batchCount = 0,
58
- batchOffset = 1,
59
- logitsDtype = 'f32',
55
+ batchCount,
56
+ batchOffset,
57
+ logitsDtype,
60
58
  } = options;
61
59
 
60
+ if (!Number.isFinite(batchCount) || batchCount < 0) {
61
+ throw new Error('[RepPenalty] batchCount is required and must be non-negative.');
62
+ }
63
+ if (!Number.isFinite(batchOffset) || batchOffset < 0) {
64
+ throw new Error('[RepPenalty] batchOffset is required and must be non-negative.');
65
+ }
66
+ if (logitsDtype !== 'f16' && logitsDtype !== 'f32') {
67
+ throw new Error('[RepPenalty] logitsDtype must be "f16" or "f32".');
68
+ }
69
+
62
70
  if (penalty === 1.0 || (historyCount === 0 && batchCount === 0)) {
63
71
  return;
64
72
  }
@@ -69,7 +77,7 @@ export async function recordRepPenalty(
69
77
  throw new Error('[RepPenalty] F16 logits requested but shader-f16 is unavailable.');
70
78
  }
71
79
 
72
- const variant = resolveVariant(useF16);
80
+ const variant = selectRepPenaltyVariant(useF16);
73
81
  const layout = getRepPenaltyBindGroupLayout(device);
74
82
  const pipeline = await createPipeline('rep_penalty', variant, layout);
75
83
 
@@ -14,6 +14,7 @@ import type { ExecutionV1PoliciesSchema } from '../../config/schema/execution-v1
14
14
  /** Residual kernel options */
15
15
  export interface ResidualOptions extends OutputBufferOptions {
16
16
  useVec4?: boolean;
17
+ outputScale?: number | null;
17
18
  dataOffset?: number;
18
19
  biasOffset?: number;
19
20
  executionPolicies?: ExecutionV1PoliciesSchema | null;
@@ -108,9 +108,21 @@ function planResidualDispatch(target, size, elementsPerWorkgroup) {
108
108
  };
109
109
  }
110
110
 
111
+ function resolveResidualOutputScale(outputScale) {
112
+ if (outputScale == null) {
113
+ return 1;
114
+ }
115
+ const value = Number(outputScale);
116
+ if (!Number.isFinite(value)) {
117
+ throw new Error(`[Residual] outputScale must be finite; got "${String(outputScale)}".`);
118
+ }
119
+ return value;
120
+ }
121
+
111
122
  async function _residualAdd(target, a, b, size, options = {}) {
112
123
  const recorder = target && typeof target.beginComputePass === 'function' ? target : null;
113
124
  const { useVec4 = true, outputBuffer = null } = options;
125
+ const outputScale = resolveResidualOutputScale(options.outputScale);
114
126
  const ownsOutput = outputBuffer == null;
115
127
 
116
128
  const { a: aAligned, b: bAligned, temps } = await alignResidualInputs(
@@ -136,7 +148,7 @@ async function _residualAdd(target, a, b, size, options = {}) {
136
148
  await unifiedKernelWrapper(
137
149
  'residual', target, variant,
138
150
  [aAligned, bAligned, output],
139
- { size, scale: 1, _pad1: dispatchPlan.dispatchStride, _pad2: 0 },
151
+ { size, scale: outputScale, _pad1: dispatchPlan.dispatchStride, _pad2: 0 },
140
152
  dispatchPlan.workgroups
141
153
  );
142
154
  return createTensor(output, outputDtype, [size], 'residual_output');
@@ -4,12 +4,12 @@
4
4
  * Residual Add Kernel
5
5
  *
6
6
  * Performs element-wise addition for residual connections.
7
- * output = a + b
7
+ * output = (a + b) * scale
8
8
  */
9
9
 
10
10
  struct Uniforms {
11
11
  size: u32, // Total number of elements
12
- scale: f32, // Scale factor for add_scaled
12
+ scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
13
13
  _pad1: u32,
14
14
  _pad2: u32,
15
15
  }
@@ -28,7 +28,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
28
28
  if (idx >= u.size) {
29
29
  return;
30
30
  }
31
- output[idx] = a[idx] + b[idx];
31
+ output[idx] = (a[idx] + b[idx]) * u.scale;
32
32
  }
33
33
 
34
34
  // In-place version: output = output + b
@@ -1,19 +1,21 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
2
2
  // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
+ // AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
4
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
5
  // residual_f16.wgsl
4
6
 
5
7
  /**
6
8
  * Residual Add Kernel (F16)
7
9
  *
8
10
  * Performs element-wise addition for residual connections.
9
- * output = a + b
11
+ * output = (a + b) * scale
10
12
  */
11
13
 
12
14
  enable f16;
13
15
 
14
16
  struct Uniforms {
15
17
  size: u32, // Total number of elements
16
- scale: f32, // Scale factor for add_scaled
18
+ scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
17
19
  _pad1: u32,
18
20
  _pad2: u32,
19
21
  }
@@ -32,5 +34,5 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
32
34
  if (idx >= u.size) {
33
35
  return;
34
36
  }
35
- output[idx] = a[idx] + b[idx];
37
+ output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
36
38
  }
@@ -10,7 +10,7 @@ enable f16;
10
10
 
11
11
  struct Uniforms {
12
12
  size: u32, // Total number of elements
13
- scale: f32, // Scale factor (unused here, kept for layout parity)
13
+ scale: f32, // Output scale
14
14
  _pad1: u32,
15
15
  _pad2: u32,
16
16
  }
@@ -36,13 +36,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
36
36
  let remaining = min(4u, size - idx);
37
37
 
38
38
  if (remaining >= 4u) {
39
- output[idx] = a[idx] + b[idx];
40
- output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
41
- output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
42
- output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
39
+ output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
40
+ output[idx + 1u] = f16(f32(a[idx + 1u] + b[idx + 1u]) * u.scale);
41
+ output[idx + 2u] = f16(f32(a[idx + 2u] + b[idx + 2u]) * u.scale);
42
+ output[idx + 3u] = f16(f32(a[idx + 3u] + b[idx + 3u]) * u.scale);
43
43
  } else {
44
44
  for (var i = 0u; i < remaining; i = i + 1u) {
45
- output[idx + i] = a[idx + i] + b[idx + i];
45
+ output[idx + i] = f16(f32(a[idx + i] + b[idx + i]) * u.scale);
46
46
  }
47
47
  }
48
48
  }
@@ -8,7 +8,7 @@
8
8
 
9
9
  struct Uniforms {
10
10
  size: u32, // Total number of elements
11
- scale: f32, // Scale factor (unused here, kept for layout parity)
11
+ scale: f32, // Output scale
12
12
  _pad1: u32,
13
13
  _pad2: u32,
14
14
  }
@@ -35,13 +35,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
35
35
  let remaining = min(4u, size - idx);
36
36
 
37
37
  if (remaining >= 4u) {
38
- output[idx] = a[idx] + b[idx];
39
- output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
40
- output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
41
- output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
38
+ output[idx] = (a[idx] + b[idx]) * u.scale;
39
+ output[idx + 1u] = (a[idx + 1u] + b[idx + 1u]) * u.scale;
40
+ output[idx + 2u] = (a[idx + 2u] + b[idx + 2u]) * u.scale;
41
+ output[idx + 3u] = (a[idx + 3u] + b[idx + 3u]) * u.scale;
42
42
  } else {
43
43
  for (var i = 0u; i < remaining; i = i + 1u) {
44
- output[idx + i] = a[idx + i] + b[idx + i];
44
+ output[idx + i] = (a[idx + i] + b[idx + i]) * u.scale;
45
45
  }
46
46
  }
47
47
  }
@@ -0,0 +1,39 @@
1
+ import type { Tensor } from '../tensor.js';
2
+ import type { CommandRecorder } from '../command-recorder.js';
3
+ import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
4
+
5
+ export declare const RMSNORM_PAIR_CACHE_LIMIT: number;
6
+
7
+ export interface SandwichRMSNormPairOptions {
8
+ batchSize?: number;
9
+ hiddenSize?: number | null;
10
+ rmsNormWeightOffset?: boolean;
11
+ label?: string | null;
12
+ layerIdx?: number;
13
+ postOutputBuffer?: GPUBuffer | null;
14
+ preOutputBuffer?: GPUBuffer | null;
15
+ }
16
+
17
+ export interface SandwichRMSNormPairResult {
18
+ postAttn: Tensor;
19
+ ffnInput: Tensor;
20
+ }
21
+
22
+ export declare function runSandwichRMSNormPair(
23
+ input: Tensor,
24
+ residual: Tensor | null,
25
+ postWeight: GPUBuffer | WeightBuffer | TensorLike,
26
+ preWeight: GPUBuffer | WeightBuffer | TensorLike,
27
+ eps: number,
28
+ options?: SandwichRMSNormPairOptions
29
+ ): Promise<SandwichRMSNormPairResult>;
30
+
31
+ export declare function recordSandwichRMSNormPair(
32
+ recorder: CommandRecorder,
33
+ input: Tensor,
34
+ residual: Tensor | null,
35
+ postWeight: GPUBuffer | WeightBuffer | TensorLike,
36
+ preWeight: GPUBuffer | WeightBuffer | TensorLike,
37
+ eps: number,
38
+ options?: SandwichRMSNormPairOptions
39
+ ): Promise<SandwichRMSNormPairResult>;
@@ -0,0 +1,152 @@
1
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
2
+ import { padToQ4KBlock } from '../../config/schema/index.js';
3
+ import { createTensor } from '../tensor.js';
4
+ import { getBuffer } from '../weight-buffer.js';
5
+ import { getKernelCapabilities } from '../device.js';
6
+ import { unifiedKernelWrapper } from './utils.js';
7
+ import {
8
+ assertRMSNormWeightBuffer,
9
+ planRMSNormDispatch,
10
+ resolveNormWeightDtype,
11
+ } from './rmsnorm.js';
12
+
13
+ export const RMSNORM_PAIR_CACHE_LIMIT = 4608;
14
+
15
+ function inferHiddenSize(input, hiddenSize) {
16
+ if (hiddenSize != null) return hiddenSize;
17
+ const shape = input?.shape;
18
+ if (Array.isArray(shape) && shape.length > 0) {
19
+ return shape[shape.length - 1];
20
+ }
21
+ return null;
22
+ }
23
+
24
+ function resolveVariant() {
25
+ const caps = getKernelCapabilities();
26
+ return caps?.hasSubgroups ? 'subgroup' : 'default';
27
+ }
28
+
29
+ function resolveDispatchLabel(label) {
30
+ if (typeof label !== 'string' || label.length === 0) {
31
+ return 'rmsnorm_pair';
32
+ }
33
+ const normalized = label.replace(/^L\d+\./, '').replace(/\s+/g, '_');
34
+ return `rmsnorm_pair:${normalized}`;
35
+ }
36
+
37
+ function assertRMSNormPairInputs(input, residual, hiddenSize) {
38
+ if (input?.dtype !== 'f32') {
39
+ throw new Error(`[rmsnorm_pair] input dtype must be f32; got "${String(input?.dtype)}".`);
40
+ }
41
+ if (residual && residual.dtype !== 'f32') {
42
+ throw new Error(`[rmsnorm_pair] residual dtype must be f32; got "${String(residual.dtype)}".`);
43
+ }
44
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0) {
45
+ throw new Error(`[rmsnorm_pair] hiddenSize must be a positive integer; got "${String(hiddenSize)}".`);
46
+ }
47
+ if (hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
48
+ throw new Error(
49
+ `[rmsnorm_pair] hiddenSize=${hiddenSize} exceeds cache limit ${RMSNORM_PAIR_CACHE_LIMIT}.`
50
+ );
51
+ }
52
+ }
53
+
54
+ async function runRMSNormPairImpl(
55
+ target,
56
+ input,
57
+ residual,
58
+ postWeight,
59
+ preWeight,
60
+ eps,
61
+ options = {}
62
+ ) {
63
+ const {
64
+ batchSize = 1,
65
+ hiddenSize = null,
66
+ rmsNormWeightOffset = false,
67
+ label = null,
68
+ postOutputBuffer = null,
69
+ preOutputBuffer = null,
70
+ } = options;
71
+ const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
72
+ assertRMSNormPairInputs(input, residual, inferredHiddenSize);
73
+
74
+ const postWeightBuffer = getBuffer(postWeight);
75
+ const preWeightBuffer = getBuffer(preWeight);
76
+ assertRMSNormWeightBuffer(postWeight, postWeightBuffer, inferredHiddenSize);
77
+ assertRMSNormWeightBuffer(preWeight, preWeightBuffer, inferredHiddenSize);
78
+ const postWeightDtype = resolveNormWeightDtype(postWeight, inferredHiddenSize);
79
+ const preWeightDtype = resolveNormWeightDtype(preWeight, inferredHiddenSize);
80
+
81
+ const paddedHiddenSize = padToQ4KBlock(inferredHiddenSize);
82
+ const outputSize = batchSize * paddedHiddenSize * 4;
83
+ const ownsPostOutput = postOutputBuffer == null;
84
+ const ownsPreOutput = preOutputBuffer == null;
85
+ const postOutput = postOutputBuffer ?? acquireBuffer(outputSize, undefined, 'rmsnorm_pair_post_output');
86
+ const preOutput = preOutputBuffer ?? acquireBuffer(outputSize, undefined, 'rmsnorm_pair_pre_output');
87
+ const residualBuffer = residual?.buffer || residual || input.buffer;
88
+ const dispatchPlan = planRMSNormDispatch(target, batchSize);
89
+ const variant = resolveVariant();
90
+ let completed = false;
91
+
92
+ try {
93
+ await unifiedKernelWrapper(
94
+ 'rmsnorm_pair',
95
+ target,
96
+ variant,
97
+ [input, residualBuffer, postWeightBuffer, preWeightBuffer, postOutput, preOutput],
98
+ {
99
+ hidden_size: inferredHiddenSize,
100
+ num_tokens: batchSize,
101
+ eps,
102
+ has_residual: residual ? 1 : 0,
103
+ token_stride: dispatchPlan.tokenStride,
104
+ _pad0: 0,
105
+ _pad1: 0,
106
+ _pad2: 0,
107
+ },
108
+ dispatchPlan.workgroups,
109
+ {
110
+ RMS_NORM_OFFSET: rmsNormWeightOffset,
111
+ POST_WEIGHT_IS_F16: postWeightDtype === 'f16',
112
+ PRE_WEIGHT_IS_F16: preWeightDtype === 'f16',
113
+ },
114
+ null,
115
+ resolveDispatchLabel(label)
116
+ );
117
+
118
+ completed = true;
119
+ return {
120
+ postAttn: createTensor(postOutput, 'f32', [batchSize, inferredHiddenSize], 'rmsnorm_pair_post_output'),
121
+ ffnInput: createTensor(preOutput, 'f32', [batchSize, inferredHiddenSize], 'rmsnorm_pair_pre_output'),
122
+ };
123
+ } finally {
124
+ if (!completed) {
125
+ if (ownsPostOutput) releaseBuffer(postOutput);
126
+ if (ownsPreOutput) releaseBuffer(preOutput);
127
+ }
128
+ }
129
+ }
130
+
131
+ export async function runSandwichRMSNormPair(
132
+ input,
133
+ residual,
134
+ postWeight,
135
+ preWeight,
136
+ eps,
137
+ options = {}
138
+ ) {
139
+ return runRMSNormPairImpl(null, input, residual, postWeight, preWeight, eps, options);
140
+ }
141
+
142
+ export async function recordSandwichRMSNormPair(
143
+ recorder,
144
+ input,
145
+ residual,
146
+ postWeight,
147
+ preWeight,
148
+ eps,
149
+ options = {}
150
+ ) {
151
+ return runRMSNormPairImpl(recorder, input, residual, postWeight, preWeight, eps, options);
152
+ }
@@ -0,0 +1,41 @@
1
+ import type { Tensor } from '../tensor.js';
2
+ import type { CommandRecorder } from '../command-recorder.js';
3
+ import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
4
+
5
+ export interface RMSNormQKOptions {
6
+ numTokens: number;
7
+ numHeads: number;
8
+ numKVHeads: number;
9
+ headDim: number;
10
+ rmsNormWeightOffset?: boolean;
11
+ }
12
+
13
+ export interface RMSNormQKResult {
14
+ q: Tensor;
15
+ k: Tensor;
16
+ }
17
+
18
+ export function canUseRMSNormQK(
19
+ qTensor: Tensor | null | undefined,
20
+ kTensor: Tensor | null | undefined,
21
+ options?: { skipKNorm?: boolean }
22
+ ): boolean;
23
+
24
+ export declare function runRMSNormQK(
25
+ qTensor: Tensor,
26
+ kTensor: Tensor,
27
+ qWeight: GPUBuffer | WeightBuffer | TensorLike,
28
+ kWeight: GPUBuffer | WeightBuffer | TensorLike,
29
+ eps: number,
30
+ options: RMSNormQKOptions
31
+ ): Promise<RMSNormQKResult>;
32
+
33
+ export declare function recordRMSNormQK(
34
+ recorder: CommandRecorder,
35
+ qTensor: Tensor,
36
+ kTensor: Tensor,
37
+ qWeight: GPUBuffer | WeightBuffer | TensorLike,
38
+ kWeight: GPUBuffer | WeightBuffer | TensorLike,
39
+ eps: number,
40
+ options: RMSNormQKOptions
41
+ ): Promise<RMSNormQKResult>;