doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -13,7 +13,7 @@ import {
13
13
  } from '../../../../gpu/weight-buffer.js';
14
14
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
15
15
  import { getRuntimeConfig } from '../../../../config/runtime.js';
16
- import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
16
+ import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
17
17
  import {
18
18
  runFusedFFN,
19
19
  recordFusedFFN,
@@ -23,8 +23,8 @@ import {
23
23
  recordCastF32ToF16,
24
24
  isFusedQ4KDisabled
25
25
  } from '../../../../gpu/kernel-selector.js';
26
- import { log, trace } from '../../../../debug/index.js';
27
- import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
26
+ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
27
+ import { isKernelDebugEnabled, dumpTokenVector, decodeReadback, getLogitsHealth, shouldDebugLayerOutput } from '../debug-utils.js';
28
28
  import { applyLoRA } from '../lora-apply.js';
29
29
  import { getLoRAModule } from '../lora.js';
30
30
  import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
@@ -54,6 +54,22 @@ function isQ4KMatmulVariant(variant) {
54
54
  return typeof variant === 'string' && variant.startsWith('q4_');
55
55
  }
56
56
 
57
+ function enqueueRecordedDenseHealth(context, layerIdx, label, tensor, elementCount) {
58
+ const recorder = context.recorder ?? null;
59
+ if (!recorder || !isTraceEnabled('logits') || !shouldDebugLayerOutput(layerIdx, context.debugLayers)) {
60
+ return;
61
+ }
62
+ if (!tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
63
+ return;
64
+ }
65
+ const dtype = tensor.dtype;
66
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
67
+ recorder.enqueueCompletionTask(async () => {
68
+ const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
69
+ trace.logits(`L${layerIdx}.${label}_HEALTH`, getLogitsHealth(decodeReadback(data, dtype)));
70
+ });
71
+ }
72
+
57
73
  export function resolveDenseFFNMatmulStepDtype(options = {}) {
58
74
  const precision = getKernelPathMatmulPrecision(
59
75
  options.role,
@@ -170,6 +186,19 @@ export function canUseNativeF16FusedGateUp(options = {}) {
170
186
  return options.gateDtype === 'f16' || options.gateDtype === 'q4k';
171
187
  }
172
188
 
189
+ function isWideTileQ4KPhaseEnabled(session, phase) {
190
+ return phase === 'decode'
191
+ ? session?.useWideTileQ4KDecode === true
192
+ : session?.useWideTileQ4KPrefill === true;
193
+ }
194
+
195
+ function shouldMarkWideTileResidualFused(session, phase) {
196
+ return getKernelCapabilities().hasF16 === true
197
+ && session?.useWideTileResidualFusion === true
198
+ && session?.retainQ4KMaterialization === true
199
+ && isWideTileQ4KPhaseEnabled(session, phase);
200
+ }
201
+
173
202
  async function coerceTensorDtype(tensor, targetDtype, recorder, options = {}) {
174
203
  if (!targetDtype || tensor.dtype === targetDtype) {
175
204
  return tensor;
@@ -419,6 +448,7 @@ export async function runDenseFFNGPU(
419
448
  dtype: gateUpOutput.dtype,
420
449
  });
421
450
  }
451
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_gate_up', gateUpOutput, numTokens * intermediateSize * 2);
422
452
 
423
453
  if (!isGpuBufferInstance(layerWeights.gateUp) && !isWeightBuffer(layerWeights.gateUp)) {
424
454
  releaseOrTrack(recorder, isWeightBuffer(gateUpWeight) ? gateUpWeight.buffer : gateUpWeight);
@@ -441,6 +471,7 @@ export async function runDenseFFNGPU(
441
471
  dtype: activatedOutput.dtype,
442
472
  });
443
473
  }
474
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
444
475
 
445
476
  if (recorder) {
446
477
  recorder.trackTemporaryBuffer(gateUpOutput.buffer);
@@ -459,7 +490,6 @@ export async function runDenseFFNGPU(
459
490
  const mergedSession = getRuntimeConfig().inference?.session;
460
491
  const tryFuseDownResidual = pendingResidual != null
461
492
  && !downLoraProbe
462
- && numTokens > 1
463
493
  && activatedOutput.dtype === 'f32'
464
494
  && downOutputDtype === 'f32'
465
495
  && pendingResidual.dtype === 'f32';
@@ -486,10 +516,7 @@ export async function runDenseFFNGPU(
486
516
  // dense.js paths; this local signal is enough for correctness.)
487
517
  {
488
518
  if (tryFuseDownResidual
489
- && getKernelCapabilities().hasF16 === true
490
- && mergedSession?.useWideTileResidualFusion === true
491
- && mergedSession?.useWideTileQ4KPrefill === true
492
- && mergedSession?.retainQ4KMaterialization === true
519
+ && shouldMarkWideTileResidualFused(mergedSession, phase)
493
520
  ) {
494
521
  residualFusedHere = true;
495
522
  context.__ffnResidualFusedFired = true;
@@ -525,6 +552,7 @@ export async function runDenseFFNGPU(
525
552
  dtype: output.dtype,
526
553
  });
527
554
  }
555
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_down', output, numTokens * hiddenSize);
528
556
 
529
557
  if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
530
558
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
@@ -636,6 +664,7 @@ export async function runDenseFFNGPU(
636
664
  hiddenActivation, swigluLimit, recorder,
637
665
  executionPolicies: context.executionPolicies ?? null,
638
666
  });
667
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
639
668
 
640
669
  let downInput = fusedOutput;
641
670
  if (downInputDtype && fusedOutput.dtype !== downInputDtype) {
@@ -662,7 +691,6 @@ export async function runDenseFFNGPU(
662
691
  const mergedSessionFused = getRuntimeConfig().inference?.session;
663
692
  const tryFuseDownResidualFused = pendingResidualFused != null
664
693
  && !downLoraProbeFused
665
- && numTokens > 1
666
694
  && downInput.dtype === 'f32'
667
695
  && fusedDownOutputDtype === 'f32'
668
696
  && pendingResidualFused.dtype === 'f32';
@@ -684,12 +712,10 @@ export async function runDenseFFNGPU(
684
712
  },
685
713
  recorder
686
714
  );
715
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_down', output, numTokens * hiddenSize);
687
716
  {
688
717
  if (tryFuseDownResidualFused
689
- && getKernelCapabilities().hasF16 === true
690
- && mergedSessionFused?.useWideTileResidualFusion === true
691
- && mergedSessionFused?.useWideTileQ4KPrefill === true
692
- && mergedSessionFused?.retainQ4KMaterialization === true
718
+ && shouldMarkWideTileResidualFused(mergedSessionFused, phase)
693
719
  ) {
694
720
  context.__ffnResidualFusedFired = true;
695
721
  }
@@ -884,6 +910,7 @@ export async function runDenseFFNGPU(
884
910
  },
885
911
  recorder
886
912
  );
913
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_down', outFused, numTokens * hiddenSize);
887
914
  if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
888
915
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
889
916
  }
@@ -1014,6 +1041,8 @@ export async function runDenseFFNGPU(
1014
1041
  operatorDiagnostics: context.operatorDiagnostics,
1015
1042
  dtype: upOutput.dtype,
1016
1043
  });
1044
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_gate', gateOutput, numTokens * intermediateSize);
1045
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_up', upOutput, numTokens * intermediateSize);
1017
1046
 
1018
1047
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1019
1048
  size: numTokens * intermediateSize,
@@ -1041,6 +1070,7 @@ export async function runDenseFFNGPU(
1041
1070
  operatorDiagnostics: context.operatorDiagnostics,
1042
1071
  dtype: activatedOutput.dtype,
1043
1072
  });
1073
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_act', activatedOutput, numTokens * intermediateSize);
1044
1074
 
1045
1075
  if (recorder) {
1046
1076
  recorder.trackTemporaryBuffer(gateOutput.buffer);
@@ -1085,6 +1115,7 @@ export async function runDenseFFNGPU(
1085
1115
  },
1086
1116
  recorder
1087
1117
  );
1118
+ enqueueRecordedDenseHealth(context, layerIdx, 'ffn_down', output, numTokens * hiddenSize);
1088
1119
 
1089
1120
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
1090
1121
  if (loraDown) {
@@ -17,5 +17,8 @@ export declare function runMoEFFNGPU(
17
17
  layerIdx: number,
18
18
  inputTensor: Tensor,
19
19
  numTokens: number,
20
- context: LayerContext
20
+ context: LayerContext,
21
+ options?: {
22
+ routerInputTensor?: Tensor | null;
23
+ }
21
24
  ): Promise<Tensor>;
@@ -7,7 +7,8 @@ export async function runMoEFFNGPU(
7
7
  layerIdx,
8
8
  inputTensor,
9
9
  numTokens,
10
- context
10
+ context,
11
+ options = {}
11
12
  ) {
12
13
  const { config, moeRouter, expertWeights, expertLoader, layerRouterWeights } = context;
13
14
 
@@ -27,15 +28,21 @@ export async function runMoEFFNGPU(
27
28
  inputTensor.buffer,
28
29
  numTokens,
29
30
  {
30
- modelType: config.modelType ?? (config.expertFormat === 'gpt-oss' ? 'gpt-oss' : null),
31
+ modelType: config.modelType ?? (
32
+ config.expertFormat === 'gpt-oss' ? 'gpt-oss' : (config.expertFormat === 'mixtral' ? 'mixtral' : 'gemma4')
33
+ ),
31
34
  hiddenSize: config.hiddenSize,
32
35
  intermediateSize: config.intermediateSize,
36
+ rmsNormEps: config.rmsNormEps,
37
+ expertIntermediateSize: config.moeExpertIntermediateSize,
33
38
  numExperts: config.numExperts,
34
39
  moeTopK: config.moeTopK,
35
40
  expertFormat: config.expertFormat,
36
41
  hiddenActivation: config.hiddenActivation,
37
42
  swigluLimit: config.swigluLimit,
38
43
  activationDtype: inputTensor.dtype,
44
+ routerInputBuffer: options.routerInputTensor?.buffer ?? null,
45
+ routerInputDtype: options.routerInputTensor?.dtype ?? null,
39
46
  kernelPath: context.kernelPath ?? null,
40
47
  executionPolicies: context.executionPolicies ?? null,
41
48
  },
@@ -21,5 +21,7 @@ export declare function processFFNWithSandwichNorm(
21
21
  size: number,
22
22
  context: LayerContext,
23
23
  layerWeights: LayerWeights | undefined,
24
- sandwichNorm: SandwichNormInfo
24
+ sandwichNorm: SandwichNormInfo,
25
+ finalOutputScale?: number | null,
26
+ precomputedFfnInput?: Tensor | null
25
27
  ): Promise<Tensor>;
@@ -1,6 +1,6 @@
1
1
 
2
2
 
3
- import { doRMSNorm, doResidualAdd, releaseOrTrack } from '../ops.js';
3
+ import { doRMSNorm, doResidualAdd, doSandwichRMSNormPair, releaseOrTrack } from '../ops.js';
4
4
  import { getLayout, getWeightDtype, isCpuWeightBuffer, isGpuBufferInstance, isWeightBuffer } from '../../../../gpu/weight-buffer.js';
5
5
  import { trace } from '../../../../debug/index.js';
6
6
  import { isKernelDebugEnabled, dumpTokenVector, logFFN, getBufferStats } from '../debug-utils.js';
@@ -19,6 +19,135 @@ async function debugFFNBuffer(context, layerIdx, label, tensor, numTokens, hidde
19
19
  await context.debugCheckBuffer(tensor.buffer, `L${layerIdx} ${label} (GPU)`, numTokens, hiddenSize);
20
20
  }
21
21
 
22
+ function shouldUseDensePlusMoeFFN(layerIdx, config, layerWeights) {
23
+ return config.ffnBranchMode === 'dense_plus_moe'
24
+ && config.useMoE
25
+ && isMoELayerLocal(layerIdx, config, layerWeights);
26
+ }
27
+
28
+ function shouldUseLegacyMoeOnlyFFN(layerIdx, config, layerWeights) {
29
+ return config.ffnBranchMode !== 'dense'
30
+ && config.ffnBranchMode !== 'dense_plus_moe'
31
+ && config.useMoE
32
+ && isMoELayerLocal(layerIdx, config, layerWeights);
33
+ }
34
+
35
+ function requireNormWeight(layerIdx, layerWeights, key) {
36
+ const weight = layerWeights?.[key];
37
+ if (!weight) {
38
+ throw new Error(`Layer ${layerIdx} dense_plus_moe FFN is missing ${key}. Re-convert the model.`);
39
+ }
40
+ return weight;
41
+ }
42
+
43
+ async function applyBranchRMSNorm(layerIdx, tensor, layerWeights, key, label, context, numTokens, hiddenSize) {
44
+ const { config, weightConfig, debugFlags, recorder } = context;
45
+ const weight = requireNormWeight(layerIdx, layerWeights, key);
46
+ const normWeightBuf = getNormWeightBuffer(weight, label, weightConfig, debugFlags);
47
+ const output = await doRMSNorm(tensor, normWeightBuf, config.rmsNormEps, {
48
+ batchSize: numTokens,
49
+ hiddenSize,
50
+ label: `L${layerIdx}.${label}`,
51
+ layerIdx,
52
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
53
+ }, recorder);
54
+ if (!isGpuBufferInstance(weight) && !isWeightBuffer(weight)) {
55
+ releaseOrTrack(recorder, normWeightBuf);
56
+ }
57
+ return output;
58
+ }
59
+
60
+ async function runDensePlusMoeFFN(
61
+ layerIdx,
62
+ postAttn,
63
+ ffnInput,
64
+ numTokens,
65
+ size,
66
+ context,
67
+ layerWeights,
68
+ decodeOutputBuffer,
69
+ finalOutputScale
70
+ ) {
71
+ const { config, recorder, weightConfig, debugFlags } = context;
72
+ const { hiddenSize, rmsNormEps } = config;
73
+ const denseRaw = await runDenseFFNGPU(layerIdx, ffnInput, numTokens, context, layerWeights);
74
+ const denseNorm = await applyBranchRMSNorm(
75
+ layerIdx,
76
+ denseRaw,
77
+ layerWeights,
78
+ 'postFeedforwardNorm1',
79
+ 'post_feedforward_norm_1',
80
+ context,
81
+ numTokens,
82
+ hiddenSize
83
+ );
84
+ releaseOrTrack(recorder, denseRaw.buffer, context.decodeBuffers);
85
+
86
+ const expertInput = await applyBranchRMSNorm(
87
+ layerIdx,
88
+ postAttn,
89
+ layerWeights,
90
+ 'preFeedforwardNorm2',
91
+ 'pre_feedforward_norm_2',
92
+ context,
93
+ numTokens,
94
+ hiddenSize
95
+ );
96
+ const expertRaw = await runMoEFFNGPU(layerIdx, expertInput, numTokens, context, {
97
+ routerInputTensor: postAttn,
98
+ });
99
+ releaseOrTrack(recorder, expertInput.buffer, context.decodeBuffers);
100
+
101
+ const expertNorm = await applyBranchRMSNorm(
102
+ layerIdx,
103
+ expertRaw,
104
+ layerWeights,
105
+ 'postFeedforwardNorm2',
106
+ 'post_feedforward_norm_2',
107
+ context,
108
+ numTokens,
109
+ hiddenSize
110
+ );
111
+ releaseOrTrack(recorder, expertRaw.buffer, context.decodeBuffers);
112
+
113
+ const combined = await doResidualAdd(denseNorm, expertNorm, size, recorder, {
114
+ label: `L${layerIdx}.ffn_dense_plus_moe`,
115
+ layerIdx,
116
+ executionPolicies: context.executionPolicies ?? null,
117
+ });
118
+ releaseOrTrack(recorder, denseNorm.buffer, context.decodeBuffers);
119
+ releaseOrTrack(recorder, expertNorm.buffer, context.decodeBuffers);
120
+
121
+ const finalNormWeight = requireNormWeight(layerIdx, layerWeights, 'postFeedforwardNorm');
122
+ const finalNormWeightBuf = getNormWeightBuffer(finalNormWeight, 'post_feedforward_norm', weightConfig, debugFlags);
123
+ const rmsNormOutputScale = finalOutputScale !== 1
124
+ && combined.dtype === context.activationDtype
125
+ && !context.debugProbes?.length
126
+ ? finalOutputScale
127
+ : 1;
128
+ const output = await doRMSNorm(combined, finalNormWeightBuf, rmsNormEps, {
129
+ batchSize: numTokens,
130
+ hiddenSize,
131
+ residual: postAttn,
132
+ outputBuffer: decodeOutputBuffer,
133
+ outputScale: rmsNormOutputScale,
134
+ label: `L${layerIdx}.post_ffn_norm`,
135
+ layerIdx,
136
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
137
+ }, recorder);
138
+ if (rmsNormOutputScale !== 1) {
139
+ context.__layerScalarFusedFired = true;
140
+ }
141
+ if (!isGpuBufferInstance(finalNormWeight) && !isWeightBuffer(finalNormWeight)) {
142
+ releaseOrTrack(recorder, finalNormWeightBuf);
143
+ }
144
+
145
+ return {
146
+ ffnOutput: combined,
147
+ output,
148
+ };
149
+ }
150
+
22
151
 
23
152
  export async function processFFNWithSandwichNorm(
24
153
  layerIdx,
@@ -27,10 +156,17 @@ export async function processFFNWithSandwichNorm(
27
156
  size,
28
157
  context,
29
158
  layerWeights,
30
- sandwichNorm
159
+ sandwichNorm,
160
+ finalOutputScale = 1,
161
+ precomputedFfnInput = null
31
162
  ) {
32
163
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
33
164
  const { hiddenSize, rmsNormEps } = config;
165
+ const requestedFinalOutputScale = finalOutputScale == null ? 1 : Number(finalOutputScale);
166
+ if (!Number.isFinite(requestedFinalOutputScale)) {
167
+ throw new Error(`Layer ${layerIdx} finalOutputScale must be finite; got "${String(finalOutputScale)}".`);
168
+ }
169
+ context.__layerScalarFusedFired = false;
34
170
 
35
171
  // For decode (M=1), get pre-allocated output buffer to avoid allocation
36
172
  const decodeOutputBuffer = numTokens === 1 && decodeBuffers
@@ -39,8 +175,8 @@ export async function processFFNWithSandwichNorm(
39
175
  const lastTokenIdx = Math.max(0, numTokens - 1);
40
176
 
41
177
  // 1. Pre-FFN norm (applied to residual stream before FFN)
42
- let ffnInput = postAttn;
43
- if (sandwichNorm.hasPreFeedforwardNorm && layerWeights?.preFeedforwardNorm) {
178
+ let ffnInput = precomputedFfnInput ?? postAttn;
179
+ if (!precomputedFfnInput && sandwichNorm.hasPreFeedforwardNorm && layerWeights?.preFeedforwardNorm) {
44
180
  const normWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
45
181
 
46
182
  ffnInput = await doRMSNorm(postAttn, normWeightBuf, rmsNormEps, {
@@ -106,9 +242,24 @@ export async function processFFNWithSandwichNorm(
106
242
  }
107
243
 
108
244
  let ffnOutput;
245
+ let combinedDenseMoeOutput = null;
109
246
  let usedFusedDownNorm = false;
110
247
 
111
- if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
248
+ if (shouldUseDensePlusMoeFFN(layerIdx, config, layerWeights)) {
249
+ const result = await runDensePlusMoeFFN(
250
+ layerIdx,
251
+ postAttn,
252
+ ffnInput,
253
+ numTokens,
254
+ size,
255
+ context,
256
+ layerWeights,
257
+ decodeOutputBuffer,
258
+ requestedFinalOutputScale
259
+ );
260
+ ffnOutput = result.ffnOutput;
261
+ combinedDenseMoeOutput = result.output;
262
+ } else if (shouldUseLegacyMoeOnlyFFN(layerIdx, config, layerWeights)) {
112
263
  ffnOutput = await runMoEFFNGPU(layerIdx, ffnInput, numTokens, context);
113
264
  } else if (canUseFusedDownNorm && layerWeights?.down && layerWeights?.postFeedforwardNorm &&
114
265
  (layerWeights?.gateUp || (layerWeights?.gate && layerWeights?.up))) {
@@ -127,6 +278,7 @@ export async function processFFNWithSandwichNorm(
127
278
  } else {
128
279
  ffnOutput = await runDenseFFNGPU(layerIdx, ffnInput, numTokens, context, layerWeights);
129
280
  }
281
+
130
282
  await runProbes('ffn_out', ffnOutput.buffer, {
131
283
  layerIdx,
132
284
  numTokens,
@@ -159,30 +311,100 @@ export async function processFFNWithSandwichNorm(
159
311
  // 3. Post-FFN norm
160
312
 
161
313
  let output;
162
- if (usedFusedDownNorm) {
314
+ const postFfnNextInputNorm = context.__postFfnNextInputNorm ?? null;
315
+ if (combinedDenseMoeOutput) {
316
+ output = combinedDenseMoeOutput;
317
+ releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
318
+ } else if (usedFusedDownNorm) {
163
319
  output = ffnOutput;
164
320
  } else if (sandwichNorm.hasPostFeedforwardNorm && layerWeights?.postFeedforwardNorm) {
165
321
  const normWeightBuf = getNormWeightBuffer(layerWeights.postFeedforwardNorm, 'post_feedforward_norm', weightConfig, debugFlags);
322
+ const rmsNormOutputScale = requestedFinalOutputScale !== 1
323
+ && ffnOutput.dtype === context.activationDtype
324
+ && !context.debugProbes?.length
325
+ ? requestedFinalOutputScale
326
+ : 1;
166
327
 
167
- output = await doRMSNorm(ffnOutput, normWeightBuf, rmsNormEps, {
168
- batchSize: numTokens,
169
- hiddenSize,
170
- residual: postAttn,
171
- outputBuffer: decodeOutputBuffer,
172
- label: `L${layerIdx}.post_ffn_norm`,
173
- layerIdx,
174
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
175
- }, recorder);
328
+ if (postFfnNextInputNorm && rmsNormOutputScale === 1) {
329
+ if (ffnOutput.dtype !== 'f32' || postAttn.dtype !== 'f32') {
330
+ throw new Error(
331
+ 'usePostFfnNextInputRMSNormPairFusion requires f32 FFN output and residual tensors ' +
332
+ `(ffn=${String(ffnOutput.dtype)}, residual=${String(postAttn.dtype)}).`
333
+ );
334
+ }
335
+ let nextInputNormWeightBuf = null;
336
+ try {
337
+ nextInputNormWeightBuf = getNormWeightBuffer(
338
+ postFfnNextInputNorm.weight,
339
+ 'next_input_norm',
340
+ weightConfig,
341
+ debugFlags
342
+ );
343
+ const pair = await doSandwichRMSNormPair(
344
+ ffnOutput,
345
+ postAttn,
346
+ normWeightBuf,
347
+ nextInputNormWeightBuf,
348
+ rmsNormEps,
349
+ {
350
+ batchSize: numTokens,
351
+ hiddenSize,
352
+ postOutputBuffer: decodeOutputBuffer,
353
+ label: `L${layerIdx}.post_ffn_next_input_norm`,
354
+ layerIdx,
355
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
356
+ },
357
+ recorder
358
+ );
359
+ output = pair.postAttn;
360
+ context.__precomputedInputNorm = {
361
+ layerIdx: postFfnNextInputNorm.layerIdx,
362
+ tensor: pair.ffnInput,
363
+ };
364
+ } finally {
365
+ if (
366
+ nextInputNormWeightBuf
367
+ && !isGpuBufferInstance(postFfnNextInputNorm.weight)
368
+ && !isWeightBuffer(postFfnNextInputNorm.weight)
369
+ ) {
370
+ releaseOrTrack(recorder, nextInputNormWeightBuf);
371
+ }
372
+ }
373
+ } else {
374
+ output = await doRMSNorm(ffnOutput, normWeightBuf, rmsNormEps, {
375
+ batchSize: numTokens,
376
+ hiddenSize,
377
+ residual: postAttn,
378
+ outputBuffer: decodeOutputBuffer,
379
+ outputScale: rmsNormOutputScale,
380
+ label: `L${layerIdx}.post_ffn_norm`,
381
+ layerIdx,
382
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
383
+ }, recorder);
384
+ if (rmsNormOutputScale !== 1) {
385
+ context.__layerScalarFusedFired = true;
386
+ }
387
+ }
176
388
 
177
389
  if (!isGpuBufferInstance(layerWeights.postFeedforwardNorm) && !isWeightBuffer(layerWeights.postFeedforwardNorm)) releaseOrTrack(recorder, normWeightBuf);
178
390
  releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
179
391
  } else {
392
+ const residualOutputScale = requestedFinalOutputScale !== 1
393
+ && ffnOutput.dtype === context.activationDtype
394
+ && postAttn.dtype === context.activationDtype
395
+ && !context.debugProbes?.length
396
+ ? requestedFinalOutputScale
397
+ : 1;
180
398
  output = await doResidualAdd(ffnOutput, postAttn, size, recorder, {
181
399
  label: `L${layerIdx}.post_ffn_residual`,
182
400
  layerIdx,
183
401
  outputBuffer: decodeOutputBuffer,
402
+ outputScale: residualOutputScale,
184
403
  executionPolicies: context.executionPolicies ?? null,
185
404
  });
405
+ if (residualOutputScale !== 1) {
406
+ context.__layerScalarFusedFired = true;
407
+ }
186
408
  releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
187
409
  }
188
410
 
@@ -19,5 +19,7 @@ export declare function processFFNStandard(
19
19
  numTokens: number,
20
20
  size: number,
21
21
  context: LayerContext,
22
- layerWeights: LayerWeights | undefined
22
+ layerWeights: LayerWeights | undefined,
23
+ fusedResidualInput?: Tensor | null,
24
+ finalOutputScale?: number | null
23
25
  ): Promise<Tensor>;
@@ -6,9 +6,11 @@ import { runProbes } from '../probes.js';
6
6
  import { isMoELayerLocal } from './types.js';
7
7
  import { runDenseFFNGPU } from './dense.js';
8
8
  import { runMoEFFNGPU } from './moe.js';
9
- import { acquireBuffer } from '../../../../memory/buffer-pool.js';
9
+ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
10
10
  import { isGpuBufferInstance, isWeightBuffer } from '../../../../gpu/weight-buffer.js';
11
- import { shouldDebugLayerOutput } from '../debug-utils/index.js';
11
+ import { shouldDebugLayerOutput, decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
12
+ import { trace, isTraceEnabled } from '../../../../debug/index.js';
13
+ import { selectRuleValue } from '../../../../rules/rule-registry.js';
12
14
 
13
15
  async function debugFFNBuffer(context, layerIdx, label, tensor, numTokens, hiddenSize) {
14
16
  if (!context.debugCheckBuffer) return;
@@ -17,6 +19,22 @@ async function debugFFNBuffer(context, layerIdx, label, tensor, numTokens, hidde
17
19
  await context.debugCheckBuffer(tensor.buffer, `L${layerIdx} ${label} (GPU)`, numTokens, hiddenSize);
18
20
  }
19
21
 
22
+ function enqueueRecordedFFNHealth(context, layerIdx, label, tensor, elementCount) {
23
+ const recorder = context.recorder ?? null;
24
+ if (!recorder || !isTraceEnabled('logits') || !shouldDebugLayerOutput(layerIdx, context.debugLayers)) {
25
+ return;
26
+ }
27
+ if (!tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
28
+ return;
29
+ }
30
+ const dtype = tensor.dtype;
31
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
32
+ recorder.enqueueCompletionTask(async () => {
33
+ const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
34
+ trace.logits(`L${layerIdx}.${label}_HEALTH`, getLogitsHealth(decodeReadback(data, dtype)));
35
+ });
36
+ }
37
+
20
38
 
21
39
  export async function processFFNStandard(
22
40
  layerIdx,
@@ -25,10 +43,16 @@ export async function processFFNStandard(
25
43
  size,
26
44
  context,
27
45
  layerWeights,
28
- fusedResidualInput
46
+ fusedResidualInput,
47
+ finalOutputScale = 1
29
48
  ) {
30
49
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
31
50
  const { hiddenSize, rmsNormEps } = config;
51
+ const requestedFinalOutputScale = finalOutputScale == null ? 1 : Number(finalOutputScale);
52
+ if (!Number.isFinite(requestedFinalOutputScale)) {
53
+ throw new Error(`Layer ${layerIdx} finalOutputScale must be finite; got "${String(finalOutputScale)}".`);
54
+ }
55
+ context.__layerScalarFusedFired = false;
32
56
 
33
57
  const decodeOutputBuffer = numTokens === 1 && decodeBuffers
34
58
  ? decodeBuffers.getOutputHiddenBuffer()
@@ -75,6 +99,7 @@ export async function processFFNStandard(
75
99
  dtype: normedTensor.dtype,
76
100
  });
77
101
  await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
102
+ enqueueRecordedFFNHealth(context, layerIdx, 'ffn_in', normedTensor, numTokens * hiddenSize);
78
103
 
79
104
  // 2. FFN
80
105
  // Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
@@ -91,7 +116,11 @@ export async function processFFNStandard(
91
116
  context.__ffnResidualFusedFired = false;
92
117
 
93
118
  let ffnOutput;
94
- if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
119
+ const useLegacyMoeOnlyFFN = config.ffnBranchMode !== 'dense'
120
+ && config.ffnBranchMode !== 'dense_plus_moe'
121
+ && config.useMoE
122
+ && isMoELayerLocal(layerIdx, config, layerWeights);
123
+ if (useLegacyMoeOnlyFFN) {
95
124
  ffnOutput = await runMoEFFNGPU(layerIdx, normedTensor, numTokens, context);
96
125
  } else {
97
126
  ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
@@ -109,6 +138,7 @@ export async function processFFNStandard(
109
138
  dtype: ffnOutput.dtype,
110
139
  });
111
140
  await debugFFNBuffer(context, layerIdx, 'FFN output', ffnOutput, numTokens, hiddenSize);
141
+ enqueueRecordedFFNHealth(context, layerIdx, 'ffn_out', ffnOutput, numTokens * hiddenSize);
112
142
 
113
143
  // 3. Residual add (uses prenorm sum when fused, otherwise postAttn)
114
144
  // When WideTile+residual fusion fired inside runDenseFFNGPU, ffnOutput
@@ -126,12 +156,21 @@ export async function processFFNStandard(
126
156
  if (residualInputOwned) {
127
157
  residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
128
158
  }
159
+ const residualOutputScale = requestedFinalOutputScale !== 1
160
+ && residualTensor.dtype === context.activationDtype
161
+ && !context.debugProbes?.length
162
+ ? requestedFinalOutputScale
163
+ : 1;
129
164
  output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
130
165
  label: `L${layerIdx}.ffn_residual`,
131
166
  layerIdx,
132
167
  outputBuffer: decodeOutputBuffer,
168
+ outputScale: residualOutputScale,
133
169
  executionPolicies: context.executionPolicies ?? null,
134
170
  });
171
+ if (residualOutputScale !== 1) {
172
+ context.__layerScalarFusedFired = true;
173
+ }
135
174
  }
136
175
  await runProbes('layer_out', output.buffer, {
137
176
  layerIdx,
@@ -143,6 +182,7 @@ export async function processFFNStandard(
143
182
  dtype: output.dtype,
144
183
  });
145
184
  await debugFFNBuffer(context, layerIdx, 'layer output', output, numTokens, hiddenSize);
185
+ enqueueRecordedFFNHealth(context, layerIdx, 'layer_out', output, numTokens * hiddenSize);
146
186
 
147
187
  if (normedTensor !== postAttn) {
148
188
  releaseOrTrack(recorder, normedTensor.buffer, decodeBuffers);