doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -1,24 +1,31 @@
1
1
  import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
2
- import { acquireBuffer, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
2
+ import { acquireBuffer, BufferUsage, releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
3
3
  import { createTensor } from '../../../gpu/tensor.js';
4
4
  import { castF16ToF32, castF32ToF16 } from '../../../gpu/kernels/cast.js';
5
5
  import {
6
6
  runMatmul,
7
7
  runSiLU,
8
+ runSiLURowSplit,
8
9
  runGeLU,
9
10
  dequantizeMXFP4Expert,
10
11
  runBiasAdd,
12
+ runRMSNorm,
13
+ runScale,
11
14
  runSoftmaxTopK,
12
15
  runMoEGather,
13
16
  runMoEBuildTokenOffsets,
14
17
  runScatterAddDynamic,
15
18
  runSwiGLURowsplitBias,
19
+ runGemma4RouteQ4MatmulF16A,
20
+ runScatterAddRoutesF16ExpertScale,
16
21
  } from '../../../gpu/kernel-selector.js';
22
+ import { getBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
17
23
  import { trace, isTraceEnabled } from '../../../debug/index.js';
18
24
  import { f16ToF32Array } from '../../kv-cache/types.js';
19
25
  import { resolveMaxTokensPerExpert, getCachedDequant, setCachedDequant, getDequantCacheStats } from './moe-cache.js';
20
26
  import { ensureExpertLoaded } from './moe-helpers.js';
21
27
  import { selectRuleValue } from '../../../rules/rule-registry.js';
28
+ import { getRuntimeConfig } from '../../../config/runtime.js';
22
29
  import {
23
30
  validateMoeShape,
24
31
  resolveMoeVendorProfile,
@@ -26,6 +33,100 @@ import {
26
33
  resolveMixtralKernelPathProfile,
27
34
  } from './moe-shape-validator.js';
28
35
  import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
36
+ import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
37
+
38
+ function resolveMoEActiveExpertSelection() {
39
+ const selection = getRuntimeConfig()?.inference?.moe?.routing?.activeExpertSelection;
40
+ if (selection === 'all' || selection === 'topk-readback' || selection === 'topk-route') {
41
+ return selection;
42
+ }
43
+ throw new Error(
44
+ '[MoE] runtime.inference.moe.routing.activeExpertSelection must be ' +
45
+ `"all", "topk-readback", or "topk-route", got ${String(selection)}.`
46
+ );
47
+ }
48
+
49
+ async function resolveActiveExpertSchedule(indicesBuffer, numTokens, numExperts, topK, maxTokensPerExpert) {
50
+ const selection = resolveMoEActiveExpertSelection();
51
+ if (selection === 'all') {
52
+ return {
53
+ selection,
54
+ activeExperts: Array.from({ length: numExperts }, (_, expertIdx) => expertIdx),
55
+ tokenCounts: null,
56
+ };
57
+ }
58
+
59
+ const indicesBytes = numTokens * topK * 4;
60
+ const indicesData = await readBuffer(indicesBuffer, indicesBytes);
61
+ return buildActiveExpertScheduleFromIndices(
62
+ new Uint32Array(indicesData),
63
+ numExperts,
64
+ maxTokensPerExpert,
65
+ selection
66
+ );
67
+ }
68
+
69
+ export function buildActiveExpertScheduleFromIndices(
70
+ indices,
71
+ numExperts,
72
+ maxTokensPerExpert,
73
+ selection = 'topk-readback'
74
+ ) {
75
+ const tokenCounts = new Uint32Array(numExperts);
76
+
77
+ for (let i = 0; i < indices.length; i++) {
78
+ const expertIdx = indices[i];
79
+ if (expertIdx >= numExperts) {
80
+ throw new Error(
81
+ `[MoE] Top-K routing produced expert index ${expertIdx} outside numExperts=${numExperts}.`
82
+ );
83
+ }
84
+ tokenCounts[expertIdx] += 1;
85
+ }
86
+
87
+ const activeExperts = [];
88
+ for (let expertIdx = 0; expertIdx < numExperts; expertIdx++) {
89
+ const count = tokenCounts[expertIdx];
90
+ if (count === 0) {
91
+ continue;
92
+ }
93
+ if (count > maxTokensPerExpert) {
94
+ throw new Error(
95
+ `[MoE] Expert ${expertIdx} received ${count} tokens but maxTokensPerExpert=${maxTokensPerExpert}. ` +
96
+ 'Increase runtime.inference.moe.routing.maxTokensPerExpert or its headroom/cap settings.'
97
+ );
98
+ }
99
+ activeExperts.push(expertIdx);
100
+ }
101
+
102
+ return { selection, activeExperts, tokenCounts };
103
+ }
104
+
105
+ function resolvePerExpertScaleBuffer(device, value) {
106
+ if (value == null) {
107
+ return { buffer: null, ownedBuffer: null };
108
+ }
109
+ if (value instanceof Float32Array) {
110
+ const buffer = acquireBuffer(value.byteLength, BufferUsage.STORAGE_READ, 'moe_per_expert_scale_f32');
111
+ try {
112
+ device.queue.writeBuffer(buffer, 0, value);
113
+ } catch (error) {
114
+ releaseBuffer(buffer);
115
+ throw error;
116
+ }
117
+ return { buffer, ownedBuffer: buffer };
118
+ }
119
+
120
+ const dtype = getWeightDtype(value);
121
+ if (dtype != null && dtype !== 'f32') {
122
+ throw new Error(`[MoE] per-expert router scale must be f32 for scatter-add, got ${dtype}.`);
123
+ }
124
+ const buffer = getBuffer(value);
125
+ if (!isGpuBufferInstance(buffer)) {
126
+ throw new Error('[MoE] per-expert router scale must resolve to a GPUBuffer.');
127
+ }
128
+ return { buffer, ownedBuffer: null };
129
+ }
29
130
 
30
131
  export async function moeFeedForwardGPU(
31
132
  inputBuffer,
@@ -40,8 +141,11 @@ export async function moeFeedForwardGPU(
40
141
  const device = getDevice();
41
142
  if (!device) throw new Error('No GPU device for MoE');
42
143
 
43
- const { hiddenSize, numExperts, intermediateSize, moeTopK, hiddenActivation } = config;
144
+ const { hiddenSize, numExperts, moeTopK, hiddenActivation } = config;
44
145
  const expertFormat = config.expertFormat;
146
+ const intermediateSize = expertFormat === 'gemma4'
147
+ ? config.expertIntermediateSize
148
+ : config.intermediateSize;
45
149
  const swigluLimit = config.swigluLimit;
46
150
  const kernelPath = config.kernelPath ?? null;
47
151
  if (!expertFormat) {
@@ -50,6 +154,9 @@ export async function moeFeedForwardGPU(
50
154
  if (swigluLimit === undefined) {
51
155
  throw new Error('MoE swigluLimit must be explicitly set (null or number).');
52
156
  }
157
+ if (!Number.isFinite(intermediateSize) || intermediateSize <= 0) {
158
+ throw new Error(`[MoE] Invalid expert intermediate size for ${expertFormat}: ${String(intermediateSize)}.`);
159
+ }
53
160
  const topK = moeTopK ?? moeRouter.topK;
54
161
  if (topK == null) {
55
162
  throw new Error('MoE topK is required in config.');
@@ -86,6 +193,12 @@ export async function moeFeedForwardGPU(
86
193
  };
87
194
 
88
195
  const inputTensor = createTensor(inputBuffer, activationDtype, [numTokens, hiddenSize], 'moe_input');
196
+ const routerSourceTensor = createTensor(
197
+ config.routerInputBuffer ?? inputBuffer,
198
+ config.routerInputDtype ?? activationDtype,
199
+ [numTokens, hiddenSize],
200
+ 'moe_router_input'
201
+ );
89
202
  let logitsBuffer = null;
90
203
  let indicesBuffer = null;
91
204
  let weightsBuffer = null;
@@ -95,18 +208,65 @@ export async function moeFeedForwardGPU(
95
208
  let expertOutputs = null;
96
209
  let tokenOffsets = null;
97
210
  let outputTensor = null;
211
+ let routerNormTensor = null;
212
+ let routerScaledTensor = null;
213
+ let ownedPerExpertScaleBuffer = null;
214
+ let activeExpertSchedule = null;
98
215
 
99
216
  const layerRouter = layerRouterWeights?.get(layerIdx) || null;
100
217
  if (layerRouter) {
101
- moeRouter.loadWeights(layerRouter.weight, layerRouter.bias || null);
218
+ moeRouter.loadWeights(
219
+ layerRouter.weight,
220
+ layerRouter.bias || null,
221
+ layerRouter.scale || null,
222
+ layerRouter.perExpertScale || null
223
+ );
102
224
  }
103
225
 
104
226
  try {
227
+ const needsGemmaRouterScale = modelType === 'diffusion_gemma'
228
+ || layerRouter?.scale != null
229
+ || layerRouter?.perExpertScale != null;
230
+ let routerInputTensor = routerSourceTensor;
231
+ if (needsGemmaRouterScale) {
232
+ if (!layerRouter?.scale) {
233
+ throw new Error(`[MoE] DiffusionGemma router scale missing for layer ${layerIdx}.`);
234
+ }
235
+ if (!layerRouter?.perExpertScale) {
236
+ throw new Error(`[MoE] DiffusionGemma per-expert router scale missing for layer ${layerIdx}.`);
237
+ }
238
+ if (!Number.isFinite(config.rmsNormEps) || config.rmsNormEps <= 0) {
239
+ throw new Error(`[MoE] DiffusionGemma router RMSNorm eps is invalid: ${String(config.rmsNormEps)}.`);
240
+ }
241
+ routerNormTensor = await runRMSNorm(
242
+ inputTensor,
243
+ layerRouter.scale,
244
+ config.rmsNormEps,
245
+ {
246
+ batchSize: numTokens,
247
+ hiddenSize,
248
+ rmsNormWeightOffset: false,
249
+ }
250
+ );
251
+ routerScaledTensor = await runScale(
252
+ routerNormTensor,
253
+ 1 / Math.sqrt(hiddenSize),
254
+ { count: numTokens * hiddenSize }
255
+ );
256
+ releaseBuffer(routerNormTensor.buffer);
257
+ routerNormTensor = null;
258
+ routerInputTensor = routerScaledTensor;
259
+ }
260
+
105
261
  let stepStart = perfMark();
106
- logitsBuffer = await moeRouter.computeRouterLogitsGPU(inputTensor.buffer, numTokens, null, {
107
- inputDtype: activationDtype,
262
+ logitsBuffer = await moeRouter.computeRouterLogitsGPU(routerInputTensor.buffer, numTokens, null, {
263
+ inputDtype: routerInputTensor.dtype,
108
264
  outputDtype: activationDtype,
109
265
  });
266
+ if (routerScaledTensor) {
267
+ releaseBuffer(routerScaledTensor.buffer);
268
+ routerScaledTensor = null;
269
+ }
110
270
  const logitsDtype = moeRouter.lastLogitsDtype ?? activationDtype;
111
271
  perfLog(`MoE L${layerIdx} router`, stepStart, { numTokens, logitsDtype });
112
272
 
@@ -227,6 +387,40 @@ export async function moeFeedForwardGPU(
227
387
  releaseBuffer(logitsBuffer);
228
388
  logitsBuffer = null;
229
389
 
390
+ const activeExpertSelection = resolveMoEActiveExpertSelection();
391
+ if (activeExpertSelection === 'topk-route') {
392
+ if (expertFormat !== 'gemma4') {
393
+ throw new Error(`[MoE] topk-route active expert selection requires gemma4 expert format, got ${expertFormat}.`);
394
+ }
395
+ stepStart = perfMark();
396
+ await ensureExpertLoaded(layerIdx, 0, expertWeights, expertLoader);
397
+ const routeWeights = expertWeights.get(`layer_${layerIdx}_expert_0`);
398
+ perfLog(`MoE L${layerIdx} route_weight_load`, stepStart, { expertFormat, topK });
399
+ stepStart = perfMark();
400
+ outputTensor = await runGemma4RouteExperts(
401
+ inputTensor,
402
+ indicesBuffer,
403
+ weightsBuffer,
404
+ layerRouter,
405
+ routeWeights,
406
+ layerIdx,
407
+ numTokens,
408
+ topK,
409
+ hiddenSize,
410
+ intermediateSize,
411
+ activationDtype,
412
+ swigluLimit
413
+ );
414
+ perfLog(`MoE L${layerIdx} route_experts`, stepStart, {
415
+ numTokens,
416
+ topK,
417
+ numRoutes: numTokens * topK,
418
+ hiddenSize,
419
+ intermediateSize,
420
+ });
421
+ return outputTensor.buffer;
422
+ }
423
+
230
424
  const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
231
425
  const bytesPerToken = hiddenSize * bytesPerElement;
232
426
  let maxTokensPerExpert = resolveMaxTokensPerExpert(numTokens, numExperts, topK, hiddenSize, activationDtype);
@@ -237,6 +431,21 @@ export async function moeFeedForwardGPU(
237
431
  );
238
432
  }
239
433
 
434
+ stepStart = perfMark();
435
+ activeExpertSchedule = await resolveActiveExpertSchedule(
436
+ indicesBuffer,
437
+ numTokens,
438
+ numExperts,
439
+ topK,
440
+ maxTokensPerExpert
441
+ );
442
+ perfLog(`MoE L${layerIdx} active_experts`, stepStart, {
443
+ selection: activeExpertSchedule.selection,
444
+ activeExperts: activeExpertSchedule.activeExperts.length,
445
+ numExperts,
446
+ maxTokensPerExpert,
447
+ });
448
+
240
449
  stepStart = perfMark();
241
450
  ({ gathered, tokenCounts, tokenMap } = await runMoEGather(
242
451
  inputTensor,
@@ -255,10 +464,6 @@ export async function moeFeedForwardGPU(
255
464
  'moe_expert_outputs_gathered'
256
465
  );
257
466
 
258
- const zeroEncoder = device.createCommandEncoder({ label: 'zero_moe_expert_outputs' });
259
- zeroEncoder.clearBuffer(expertOutputs, 0, numExperts * maxTokensPerExpert * hiddenSize * bytesPerElement);
260
- device.queue.submit([zeroEncoder.finish()]);
261
-
262
467
  stepStart = perfMark();
263
468
  tokenOffsets = await runMoEBuildTokenOffsets(
264
469
  tokenCounts,
@@ -279,11 +484,13 @@ export async function moeFeedForwardGPU(
279
484
  const expertStrideBytes = maxTokensPerExpert * bytesPerToken;
280
485
  const rowsPerExpert = maxTokensPerExpert;
281
486
 
282
- // GPU-first execution path: avoid CPU readback of tokenCounts for scheduling.
283
- // Each expert executes with a fixed row budget (maxTokensPerExpert); gathered
284
- // unused rows are zero-filled and never consumed by scatter_add_dynamic.
285
- for (let expertIdx = 0; expertIdx < numExperts; expertIdx++) {
286
- const count = rowsPerExpert;
487
+ const scheduledExperts = activeExpertSchedule?.activeExperts ?? [];
488
+ const scheduledTokenCounts = activeExpertSchedule?.tokenCounts ?? null;
489
+ for (const expertIdx of scheduledExperts) {
490
+ const count = scheduledTokenCounts ? scheduledTokenCounts[expertIdx] : rowsPerExpert;
491
+ if (count <= 0) {
492
+ continue;
493
+ }
287
494
 
288
495
  stepStart = perfMark();
289
496
  await ensureExpertLoaded(layerIdx, expertIdx, expertWeights, expertLoader);
@@ -328,6 +535,22 @@ export async function moeFeedForwardGPU(
328
535
  vendorProfile,
329
536
  gptOssKernelPathProfile
330
537
  );
538
+ } else if (expertFormat === 'gemma4' && weights.gateUp && weights.down) {
539
+ await runGemma4Expert(
540
+ gathered,
541
+ expertOutputs,
542
+ weights,
543
+ count,
544
+ inputOffset,
545
+ outputOffset,
546
+ hiddenSize,
547
+ intermediateSize,
548
+ activationDtype,
549
+ swigluLimit,
550
+ kernelPath
551
+ );
552
+ } else if (expertFormat === 'gemma4') {
553
+ throw new Error(`[MoE] Missing Gemma-style packed weights for ${expertKey}`);
331
554
  } else if (expertFormat === 'mixtral' && weights.gate && weights.up && weights.down) {
332
555
  await runMixtralExpert(
333
556
  gathered,
@@ -355,6 +578,8 @@ export async function moeFeedForwardGPU(
355
578
  [numExperts, maxTokensPerExpert, hiddenSize],
356
579
  'moe_expert_outputs'
357
580
  );
581
+ const perExpertScale = resolvePerExpertScaleBuffer(device, layerRouter?.perExpertScale || null);
582
+ ownedPerExpertScaleBuffer = perExpertScale.ownedBuffer;
358
583
  stepStart = perfMark();
359
584
  outputTensor = await runScatterAddDynamic(
360
585
  expertOutputsTensor,
@@ -364,7 +589,10 @@ export async function moeFeedForwardGPU(
364
589
  numTokens,
365
590
  hiddenSize,
366
591
  topK,
367
- { weightsDtype: activationDtype }
592
+ {
593
+ weightsDtype: activationDtype,
594
+ perExpertScale: perExpertScale.buffer,
595
+ }
368
596
  );
369
597
  perfLog(`MoE L${layerIdx} scatter`, stepStart, { numTokens, hiddenSize });
370
598
 
@@ -386,7 +614,9 @@ export async function moeFeedForwardGPU(
386
614
  trace.perf(`MoE L${layerIdx} done`, {
387
615
  numTokens,
388
616
  topK,
389
- executedExperts: numExperts,
617
+ executedExperts: scheduledExperts.length,
618
+ activeExperts: scheduledExperts.length,
619
+ activeExpertSelection: activeExpertSchedule?.selection ?? null,
390
620
  rowsPerExpert,
391
621
  maxTokensPerExpert,
392
622
  dequantCacheHits: cacheStats.hits,
@@ -400,6 +630,8 @@ export async function moeFeedForwardGPU(
400
630
  return outputTensor.buffer;
401
631
  } finally {
402
632
  if (logitsBuffer) releaseBuffer(logitsBuffer);
633
+ if (routerNormTensor?.buffer) releaseBuffer(routerNormTensor.buffer);
634
+ if (routerScaledTensor?.buffer) releaseBuffer(routerScaledTensor.buffer);
403
635
  if (tokenCounts) releaseBuffer(tokenCounts);
404
636
  if (gathered?.buffer) releaseBuffer(gathered.buffer);
405
637
  if (tokenMap) releaseBuffer(tokenMap);
@@ -407,14 +639,127 @@ export async function moeFeedForwardGPU(
407
639
  if (tokenOffsets) releaseBuffer(tokenOffsets);
408
640
  if (indicesBuffer) releaseBuffer(indicesBuffer);
409
641
  if (weightsBuffer) releaseBuffer(weightsBuffer);
642
+ if (ownedPerExpertScaleBuffer) releaseBuffer(ownedPerExpertScaleBuffer);
410
643
  }
411
644
  }
412
645
 
413
646
  function inferBufferDtype(buffer, expectedElements) {
647
+ if (isWeightBuffer(buffer)) {
648
+ return getWeightDtype(buffer);
649
+ }
650
+ const dtype = getWeightDtype(buffer);
651
+ if (dtype) return dtype;
414
652
  const bytesPerElement = Math.round(buffer.size / expectedElements);
415
653
  return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
416
654
  }
417
655
 
656
+ function alignTo4(value) {
657
+ return Math.ceil(value / 4) * 4;
658
+ }
659
+
660
+ function resolveMatrixStorageStrideBytes(buffer, rows, cols, label) {
661
+ const dtype = inferBufferDtype(buffer, rows * cols, label);
662
+ if (dtype === 'q4k') {
663
+ return alignTo4(rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES);
664
+ }
665
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
666
+ return alignTo4(rows * cols * bytesPerElement);
667
+ }
668
+
669
+ async function runGemma4RouteExperts(
670
+ inputTensor,
671
+ indicesBuffer,
672
+ weightsBuffer,
673
+ layerRouter,
674
+ weights,
675
+ layerIdx,
676
+ numTokens,
677
+ topK,
678
+ hiddenSize,
679
+ intermediateSize,
680
+ activationDtype,
681
+ swigluLimit
682
+ ) {
683
+ if (activationDtype !== 'f16') {
684
+ throw new Error(`[MoE] topk-route Gemma4 expert path requires f16 activations, got ${activationDtype}.`);
685
+ }
686
+ if (!weights?.gateUp || !weights?.down || weights.expertFormat !== 'gemma4') {
687
+ throw new Error(`[MoE] topk-route Gemma4 expert path missing packed weights for layer ${layerIdx}.`);
688
+ }
689
+ if (!layerRouter?.perExpertScale) {
690
+ throw new Error(`[MoE] topk-route Gemma4 expert path requires per-expert router scale for layer ${layerIdx}.`);
691
+ }
692
+
693
+ const device = getDevice();
694
+ const numRoutes = numTokens * topK;
695
+ const gateUpOutDim = intermediateSize * 2;
696
+ let gateUpOut = null;
697
+ let activated = null;
698
+ let routeDown = null;
699
+ let ownedPerExpertScaleBuffer = null;
700
+
701
+ try {
702
+ gateUpOut = await runGemma4RouteQ4MatmulF16A(
703
+ inputTensor,
704
+ indicesBuffer,
705
+ weights.gateUp,
706
+ {
707
+ numRoutes,
708
+ topK,
709
+ N: gateUpOutDim,
710
+ K: hiddenSize,
711
+ inputMode: 'token',
712
+ label: `moe_l${layerIdx}_route_gate_up`,
713
+ }
714
+ );
715
+ activated = await runSiLURowSplit(gateUpOut, {
716
+ numTokens: numRoutes,
717
+ dim: intermediateSize,
718
+ activation: 'gelu',
719
+ swigluLimit,
720
+ });
721
+ releaseBuffer(gateUpOut.buffer);
722
+ gateUpOut = null;
723
+
724
+ routeDown = await runGemma4RouteQ4MatmulF16A(
725
+ activated,
726
+ indicesBuffer,
727
+ weights.down,
728
+ {
729
+ numRoutes,
730
+ topK,
731
+ N: hiddenSize,
732
+ K: intermediateSize,
733
+ inputMode: 'route',
734
+ label: `moe_l${layerIdx}_route_down`,
735
+ }
736
+ );
737
+ releaseBuffer(activated.buffer);
738
+ activated = null;
739
+
740
+ const perExpertScale = resolvePerExpertScaleBuffer(device, layerRouter.perExpertScale);
741
+ ownedPerExpertScaleBuffer = perExpertScale.ownedBuffer;
742
+ const outputTensor = await runScatterAddRoutesF16ExpertScale(
743
+ routeDown,
744
+ indicesBuffer,
745
+ weightsBuffer,
746
+ perExpertScale.buffer,
747
+ numTokens,
748
+ hiddenSize,
749
+ topK,
750
+ { label: `moe_l${layerIdx}_route_scatter` }
751
+ );
752
+ releaseBuffer(routeDown.buffer);
753
+ routeDown = null;
754
+ return outputTensor;
755
+ } finally {
756
+ if (gateUpOut?.buffer) releaseBuffer(gateUpOut.buffer);
757
+ if (activated?.buffer) releaseBuffer(activated.buffer);
758
+ if (routeDown?.buffer) releaseBuffer(routeDown.buffer);
759
+ if (ownedPerExpertScaleBuffer) releaseBuffer(ownedPerExpertScaleBuffer);
760
+ }
761
+ }
762
+
418
763
  async function runGptOssExpert(
419
764
  gathered,
420
765
  expertOutputs,
@@ -587,6 +932,90 @@ async function runGptOssExpert(
587
932
  }
588
933
  }
589
934
 
935
+ async function runGemma4Expert(
936
+ gathered,
937
+ expertOutputs,
938
+ weights,
939
+ count,
940
+ inputOffset,
941
+ outputOffset,
942
+ hiddenSize,
943
+ intermediateSize,
944
+ activationDtype,
945
+ swigluLimit,
946
+ kernelPath
947
+ ) {
948
+ const numExperts = weights.numExperts;
949
+ const expertIdx = weights.expertIdx;
950
+ if (!Number.isFinite(numExperts) || numExperts <= 0) {
951
+ throw new Error(`[MoE] Gemma-style expert ${expertIdx} missing numExperts.`);
952
+ }
953
+ if (!Number.isFinite(expertIdx) || expertIdx < 0) {
954
+ throw new Error('[MoE] Gemma-style expert missing expertIdx.');
955
+ }
956
+ if (expertIdx >= numExperts) {
957
+ throw new Error(`[MoE] Gemma-style expert index ${expertIdx} out of range for ${numExperts} experts.`);
958
+ }
959
+
960
+ const gateUpOutDim = intermediateSize * 2;
961
+ const gateUpStrideBytes = resolveMatrixStorageStrideBytes(
962
+ weights.gateUp,
963
+ gateUpOutDim,
964
+ hiddenSize,
965
+ 'Gemma gate_up_proj'
966
+ );
967
+ const gateUpOffset = expertIdx * gateUpStrideBytes;
968
+ const downStrideBytes = resolveMatrixStorageStrideBytes(
969
+ weights.down,
970
+ hiddenSize,
971
+ intermediateSize,
972
+ 'Gemma down_proj'
973
+ );
974
+ const downOffset = expertIdx * downStrideBytes;
975
+
976
+ const gateUpOut = await runMatmul(
977
+ gathered,
978
+ weights.gateUp,
979
+ count,
980
+ gateUpOutDim,
981
+ hiddenSize,
982
+ {
983
+ transposeB: true,
984
+ aOffset: inputOffset,
985
+ bOffset: gateUpOffset,
986
+ outputDtype: activationDtype,
987
+ role: 'moe_gate_up',
988
+ kernelPath,
989
+ }
990
+ );
991
+
992
+ const activated = await runSiLURowSplit(gateUpOut, {
993
+ numTokens: count,
994
+ dim: intermediateSize,
995
+ activation: 'gelu',
996
+ swigluLimit,
997
+ });
998
+ releaseBuffer(gateUpOut.buffer);
999
+
1000
+ await runMatmul(
1001
+ activated,
1002
+ weights.down,
1003
+ count,
1004
+ hiddenSize,
1005
+ intermediateSize,
1006
+ {
1007
+ transposeB: true,
1008
+ bOffset: downOffset,
1009
+ outputBuffer: expertOutputs,
1010
+ cOffset: outputOffset,
1011
+ outputDtype: activationDtype,
1012
+ role: 'moe_down',
1013
+ kernelPath,
1014
+ }
1015
+ );
1016
+ releaseBuffer(activated.buffer);
1017
+ }
1018
+
590
1019
  async function runMixtralExpert(
591
1020
  gathered,
592
1021
  expertOutputs,
@@ -10,6 +10,7 @@
10
10
  * Supports multiple MoE architectures:
11
11
  * - Mixtral-style (gate/up/down per expert)
12
12
  * - GPT-OSS style (MXFP4 quantized fused gate_up + bias)
13
+ * - Gemma-style packed F16 gate_up/down experts
13
14
  *
14
15
  * @module inference/pipelines/text/moe-impl
15
16
  */
@@ -19,6 +20,7 @@ import type { MoERouter } from '../../moe-router.js';
19
20
  import type { ExpertWeights } from './types.js';
20
21
  import type { KernelPathSchema } from '../../../config/schema/index.js';
21
22
  import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
23
+ import type { WeightBuffer } from '../../../gpu/weight-buffer.js';
22
24
 
23
25
  /**
24
26
  * Clear the dequantization cache (call on model unload).
@@ -47,12 +49,16 @@ export interface MoEConfig {
47
49
  modelType?: string | null;
48
50
  hiddenSize: number;
49
51
  intermediateSize: number;
52
+ rmsNormEps?: number;
53
+ expertIntermediateSize?: number;
50
54
  numExperts: number;
51
55
  moeTopK: number;
52
- expertFormat: 'mixtral' | 'gpt-oss';
56
+ expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4';
53
57
  hiddenActivation: string;
54
58
  swigluLimit: number | null;
55
59
  activationDtype?: TensorDtype;
60
+ routerInputBuffer?: GPUBuffer | null;
61
+ routerInputDtype?: TensorDtype | null;
56
62
  kernelPath?: KernelPathSchema | null;
57
63
  executionPolicies?: ExecutionV1PoliciesSchema | null;
58
64
  }
@@ -61,8 +67,10 @@ export interface MoEConfig {
61
67
  * Expert weights with optional GPT-OSS quantized format.
62
68
  */
63
69
  export interface MoEExpertWeights extends ExpertWeights {
64
- expertFormat: 'mixtral' | 'gpt-oss';
70
+ expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4';
65
71
  numExperts?: number;
72
+ expertIntermediateSize?: number;
73
+ gateUp?: GPUBuffer;
66
74
  gateUpBlocks?: GPUBuffer;
67
75
  gateUpScales?: GPUBuffer;
68
76
  gateUpBias?: GPUBuffer;
@@ -75,8 +83,10 @@ export interface MoEExpertWeights extends ExpertWeights {
75
83
  * Layer router weights (for models with per-layer routers like GPT-OSS).
76
84
  */
77
85
  export interface LayerRouterWeights {
78
- weight: Float32Array | GPUBuffer;
86
+ weight: Float32Array | GPUBuffer | WeightBuffer;
79
87
  bias: Float32Array | GPUBuffer | null;
88
+ scale?: Float32Array | GPUBuffer | WeightBuffer | null;
89
+ perExpertScale?: Float32Array | GPUBuffer | WeightBuffer | null;
80
90
  }
81
91
 
82
92
  /**
@@ -15,6 +15,14 @@ export function resolveMoeVendorProfile(modelType) {
15
15
  if (modelType === 'mixtral') {
16
16
  return selectRuleValue('kernels', 'moeMixtral', 'vendorQuirkProfile', { vendor });
17
17
  }
18
+ if (modelType === 'gemma4' || modelType === 'diffusion_gemma') {
19
+ return {
20
+ preferVec4Dequant: false,
21
+ dequantTileShape: null,
22
+ routerWorkgroupSize: 256,
23
+ maxTokensPerExpertScale: 1.0,
24
+ };
25
+ }
18
26
  throw new Error(`[MoE] Unknown modelType "${modelType}" for vendor profile resolution.`);
19
27
  }
20
28
 
@@ -66,7 +74,11 @@ export function validateMoeShape(config, options = {}) {
66
74
  numExperts,
67
75
  expertFormat,
68
76
  } = config;
69
- const modelType = options.modelType ?? (expertFormat === 'gpt-oss' ? 'gpt-oss' : 'mixtral');
77
+ const modelType = options.modelType ?? (
78
+ expertFormat === 'gpt-oss'
79
+ ? 'gpt-oss'
80
+ : (expertFormat === 'gemma4' ? 'gemma4' : 'mixtral')
81
+ );
70
82
 
71
83
  if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
72
84
  throw new Error(`[MoE] hiddenSize must be > 0, got ${hiddenSize}.`);