doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -16,7 +16,11 @@ import type { ParsedModelConfig, Manifest } from './config.js';
16
16
  import type { KernelCapabilities } from '../../../gpu/device.js';
17
17
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
18
18
  import type { LoadedEmbeddingPostprocessor } from '../../../loader/final-weights-loader.js';
19
- import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
19
+ import type {
20
+ DiffusionGemmaSelfConditioningWeights,
21
+ LoaderLoadTiming,
22
+ PerLayerInputWeights,
23
+ } from '../../../loader/loader-types.js';
20
24
  import {
21
25
  KVCache,
22
26
  SlidingWindowKVCache,
@@ -57,6 +61,7 @@ export interface PipelineStorageContext {
57
61
  ) => AsyncIterable<ArrayBuffer | Uint8Array>;
58
62
  loadTokenizerJson?: () => Promise<Record<string, unknown> | string | null | undefined>;
59
63
  loadTokenizerModel?: (path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
64
+ loadAuxiliaryFile?: (path: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
60
65
  loadTensorsJson?: () => Promise<string | Record<string, unknown> | null | undefined>;
61
66
  verifyHashes?: boolean;
62
67
  close?: () => Promise<void>;
@@ -219,8 +224,10 @@ export interface WeightLoadResult {
219
224
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
220
225
  finalNorm: GPUBuffer | Float32Array | null;
221
226
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
227
+ diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
222
228
  perLayerInputWeights: PerLayerInputWeights | null;
223
229
  layerRouterWeights: Map<number, RouterWeights>;
230
+ loadTiming: LoaderLoadTiming | null;
224
231
  }
225
232
 
226
233
  export interface ResolvedQ4KConfig {
@@ -418,6 +418,20 @@ function usesMixedGeometryKVCache(modelConfig) {
418
418
  return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
419
419
  }
420
420
 
421
+ function resolveMixedGeometrySlidingLayerLayout(modelConfig) {
422
+ const diffusionGemmaContract = modelConfig?.diffusionGemma ?? null;
423
+ if (diffusionGemmaContract == null) {
424
+ return 'ring';
425
+ }
426
+ if (diffusionGemmaContract.decoderCacheMode === 'encoder_kv_readonly_canvas_concat') {
427
+ return 'contiguous';
428
+ }
429
+ throw new Error(
430
+ `Unsupported DiffusionGemma decoderCacheMode="${String(diffusionGemmaContract.decoderCacheMode)}" ` +
431
+ 'for mixed-geometry KV cache.'
432
+ );
433
+ }
434
+
421
435
  function getRequiredKernelMaxKVLen(operation, variant, label) {
422
436
  const config = getKernelConfig(operation, variant);
423
437
  const maxKVLen = config.variantMetadata?.maxKVLen;
@@ -693,6 +707,7 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
693
707
  globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
694
708
  globalHeadDim: modelConfig.globalHeadDim ?? null,
695
709
  slidingWindow,
710
+ slidingLayerLayout: resolveMixedGeometrySlidingLayerLayout(modelConfig),
696
711
  layerTypes: modelConfig.layerTypes,
697
712
  });
698
713
  } else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
@@ -895,6 +910,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
895
910
  verify: verifyHashes,
896
911
  loadShardRange,
897
912
  streamShardRange,
913
+ loadAuxiliaryFile: typeof runtimeStorageContext?.loadAuxiliaryFile === 'function'
914
+ ? (path) => runtimeStorageContext.loadAuxiliaryFile(path)
915
+ : null,
898
916
  });
899
917
  if (isRDRRManifest(manifest)) {
900
918
  dopplerLoader.setManifest(manifest);
@@ -942,6 +960,8 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
942
960
  layerRouterWeights.set(l, {
943
961
  weight: weights.routerWeight,
944
962
  bias: weights.routerBias || null,
963
+ scale: weights.routerScale || null,
964
+ perExpertScale: weights.routerPerExpertScale || null,
945
965
  });
946
966
  }
947
967
  }
@@ -954,8 +974,12 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
954
974
  lmHead: dopplerLoader.lmHead,
955
975
  finalNorm: dopplerLoader.finalNorm,
956
976
  embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
977
+ diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
957
978
  perLayerInputWeights: dopplerLoader.perLayerInputWeights,
958
979
  layerRouterWeights,
980
+ loadTiming: typeof dopplerLoader.getLoadTiming === 'function'
981
+ ? dopplerLoader.getLoadTiming()
982
+ : null,
959
983
  };
960
984
  }
961
985
 
@@ -978,7 +1002,12 @@ export function initMoERouter(modelConfig, moeRoutingConfig, layerWeights) {
978
1002
  for (let l = 0; l < modelConfig.numLayers; l++) {
979
1003
  const weights = layerWeights.get(`layer_${l}`);
980
1004
  if (weights?.routerWeight) {
981
- router.loadWeights(weights.routerWeight, weights.routerBias || null);
1005
+ router.loadWeights(
1006
+ weights.routerWeight,
1007
+ weights.routerBias || null,
1008
+ weights.routerScale || null,
1009
+ weights.routerPerExpertScale || null
1010
+ );
982
1011
  log.debug('Pipeline', `Loaded MoE router from layer ${l}${weights.routerBias ? ' (with bias)' : ''}`);
983
1012
  break;
984
1013
  }
@@ -24,12 +24,36 @@ import {
24
24
  hasPerLayerInputBlock,
25
25
  applyPerLayerInputBlock,
26
26
  applyLayerScalar,
27
+ resolveLayerScalarValue,
27
28
  } from './layer.js';
28
29
 
29
30
  // ============================================================================
30
31
  // Configurable Layer Pipeline (JSON-Driven)
31
32
  // ============================================================================
32
33
 
34
+ function isFinalResidualOutputStep(steps, stepIndex) {
35
+ const step = steps[stepIndex];
36
+ return step?.op === 'residual_add'
37
+ && step.dst === 'state'
38
+ && !step.probeStage
39
+ && stepIndex === steps.length - 1;
40
+ }
41
+
42
+ function resolveResidualOutputScaleForPlan(steps, stepIndex, config, context, layerWeights, residualDtype) {
43
+ if (!isFinalResidualOutputStep(steps, stepIndex)) {
44
+ return 1;
45
+ }
46
+ if (hasPerLayerInputBlock(config)) {
47
+ return 1;
48
+ }
49
+ if (residualDtype !== resolveActivationDtype(context.activationDtype)) {
50
+ return 1;
51
+ }
52
+ if (context.debugProbes?.length) {
53
+ return 1;
54
+ }
55
+ return resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
56
+ }
33
57
 
34
58
  function resolveNormWeightForPlan(weight, layerWeights) {
35
59
  if (!layerWeights) return null;
@@ -60,7 +84,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
60
84
 
61
85
  const planSteps = getLayerPlanSteps(context.pipelinePlan, layerIdx);
62
86
  const steps = filterLayerPlanStepsByPhase(planSteps, isPrefill);
63
- const device = getDevice();
87
+ const device = recorder?.device ?? getDevice();
64
88
  if (!device) throw new Error('No GPU device available');
65
89
 
66
90
  const layerType = config.layerTypes?.[layerIdx];
@@ -194,7 +218,9 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
194
218
  };
195
219
 
196
220
  try {
197
- for (const step of steps) {
221
+ let layerScalarFused = false;
222
+ for (let stepIndex = 0; stepIndex < steps.length; stepIndex++) {
223
+ const step = steps[stepIndex];
198
224
  switch (step.op) {
199
225
  case 'save': {
200
226
  const src = getSlot(step.src);
@@ -239,6 +265,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
239
265
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
240
266
  queryPreAttnScalar: config.queryPreAttnScalar,
241
267
  queryKeyNorm: config.queryKeyNorm,
268
+ queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
242
269
  valueNorm: config.valueNorm,
243
270
  attentionOutputGate: config.attentionOutputGate,
244
271
  outputGateType: config.outputGateType ?? null,
@@ -265,8 +292,8 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
265
292
  attnState,
266
293
  context.debug,
267
294
  { debugLayers: context.debugLayers },
268
- (weight, label) => getWeightBuffer(weight, label),
269
- (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
295
+ (weight, label) => getWeightBuffer(weight, label, device),
296
+ (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags, device),
270
297
  context.debugCheckBuffer,
271
298
  recorder,
272
299
  context.lora
@@ -304,9 +331,9 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
304
331
 
305
332
  const outputTensor = await doConv(
306
333
  srcTensor,
307
- getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj`),
308
- convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel`) : null,
309
- getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj`),
334
+ getWeightBuffer(convInProj, `L${layerIdx}.plan_conv_in_proj`, device),
335
+ convKernel ? getWeightBuffer(convKernel, `L${layerIdx}.plan_conv_kernel`, device) : null,
336
+ getWeightBuffer(convOutProj, `L${layerIdx}.plan_conv_out_proj`, device),
310
337
  {
311
338
  numTokens,
312
339
  hiddenSize,
@@ -340,7 +367,7 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
340
367
  if (!weight) {
341
368
  throw new Error(`Layer pipeline rmsnorm missing weights for "${step.weight}" at L${layerIdx}`);
342
369
  }
343
- const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags);
370
+ const normWeightBuf = getNormWeightBuffer(weight, `rmsnorm_${step.weight}`, weightConfig, debugFlags, device);
344
371
  const residualBuf = step.residual ? getSlot(step.residual) : null;
345
372
 
346
373
  const activationDtype = resolveStepInputDtype(step, step.src);
@@ -388,6 +415,13 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
388
415
  let outputTensor;
389
416
  const { runMoEFFNGPU, runDenseFFNGPU } = await import('./ffn/index.js');
390
417
 
418
+ if (config.ffnBranchMode === 'dense_plus_moe') {
419
+ throw new Error(
420
+ `Layer ${layerIdx} uses ffn.branchMode="dense_plus_moe", but execution-v1 generic ffn steps ` +
421
+ 'cannot express separate dense, expert, and router inputs. Use the sandwich FFN runtime path or add a dedicated plan op.'
422
+ );
423
+ }
424
+
391
425
  const canAutoMoe = config.useMoE && isMoELayer(layerIdx, config);
392
426
  const useMoe = selectRuleValue(
393
427
  'inference',
@@ -422,11 +456,23 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
422
456
  const activationDtype = resolveStepInputDtype(step, step.a ?? 'state');
423
457
  const aTensor = createTensor(aBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_a');
424
458
  const bTensor = createTensor(bBuf, activationDtype, [numTokens, hiddenSize], 'plan_residual_b');
459
+ const outputScale = resolveResidualOutputScaleForPlan(
460
+ steps,
461
+ stepIndex,
462
+ config,
463
+ context,
464
+ layerWeights,
465
+ activationDtype
466
+ );
425
467
  const outputTensor = await doResidualAdd(aTensor, bTensor, size, recorder, {
426
468
  label: `L${layerIdx}.residual_add`,
427
469
  layerIdx,
470
+ outputScale,
428
471
  executionPolicies: context.executionPolicies ?? null,
429
472
  });
473
+ if (outputScale !== 1) {
474
+ layerScalarFused = true;
475
+ }
430
476
  const outputDtype = resolveStepOutputDtype(step, resolveActivationDtype(outputTensor.dtype));
431
477
  setSlot(step.dst, outputTensor.buffer, outputDtype);
432
478
  if (step.probeStage) {
@@ -499,12 +545,14 @@ export async function processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPr
499
545
  const outputDtype = resolveActivationDtype(outputTensor.dtype);
500
546
  setSlot('state', outputTensor.buffer, outputDtype);
501
547
  }
502
- const stateBuffer = getSlot('state');
503
- const stateDtype = getSlotDtype('state') ?? activationDtype;
504
- const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state_layer_scalar');
505
- const scaledTensor = await applyLayerScalar(layerIdx, stateTensor, size, context, layerWeights);
506
- if (scaledTensor.buffer !== stateTensor.buffer) {
507
- setSlot('state', scaledTensor.buffer, resolveActivationDtype(scaledTensor.dtype));
548
+ if (!layerScalarFused) {
549
+ const stateBuffer = getSlot('state');
550
+ const stateDtype = getSlotDtype('state') ?? activationDtype;
551
+ const stateTensor = createTensor(stateBuffer, stateDtype, [numTokens, hiddenSize], 'plan_state_layer_scalar');
552
+ const scaledTensor = await applyLayerScalar(layerIdx, stateTensor, size, context, layerWeights);
553
+ if (scaledTensor.buffer !== stateTensor.buffer) {
554
+ setSlot('state', scaledTensor.buffer, resolveActivationDtype(scaledTensor.dtype));
555
+ }
508
556
  }
509
557
  } catch (err) {
510
558
  cleanupSlots();
@@ -1,13 +1,14 @@
1
1
 
2
2
 
3
3
  import { log, trace } from '../../../debug/index.js';
4
+ import { getRuntimeConfig } from '../../../config/runtime.js';
4
5
  import { getDevice } from '../../../gpu/device.js';
5
6
  import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
6
7
  import { allowReadback } from '../../../gpu/perf-guards.js';
7
8
  import { createTensor } from '../../../gpu/tensor.js';
8
9
  import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
9
10
  import {
10
- doAttention, doRMSNorm, doResidualAdd, doMatmul, doGeLU,
11
+ doAttention, doRMSNorm, doSandwichRMSNormPair, doResidualAdd, doMatmul, doGeLU,
11
12
  doConv,
12
13
  doCast,
13
14
  releaseOrTrack
@@ -21,6 +22,7 @@ import { logLayer, logAttn, getBufferStats, isKernelDebugEnabled, dumpTokenVecto
21
22
  import { runProbes } from './probes.js';
22
23
  import { selectRuleValue } from '../../../rules/rule-registry.js';
23
24
  import { recordCheckFiniteness } from '../../../gpu/kernels/check-finiteness.js';
25
+ import { RMSNORM_PAIR_CACHE_LIMIT } from '../../../gpu/kernel-selector.js';
24
26
  import { shouldRunFinitenessGuard } from './finiteness-policy.js';
25
27
  import { runLinearAttentionLayer } from './linear-attention.js';
26
28
  import { validateAttnConfig } from './attention/attn-config.js';
@@ -46,6 +48,122 @@ export function detectSandwichNorm(config) {
46
48
  };
47
49
  }
48
50
 
51
+ function shouldUseSandwichRMSNormPairFusion({
52
+ context,
53
+ sandwichNorm,
54
+ layerWeights,
55
+ numTokens,
56
+ hiddenSize,
57
+ attnOutput,
58
+ inputTensor,
59
+ }) {
60
+ const mergedSession = getRuntimeConfig()?.inference?.session;
61
+ if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
62
+ return false;
63
+ }
64
+ if (numTokens !== 1) {
65
+ return false;
66
+ }
67
+ if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostAttentionNorm || !sandwichNorm.hasPreFeedforwardNorm) {
68
+ return false;
69
+ }
70
+ if (!layerWeights?.postAttentionNorm || !layerWeights?.preFeedforwardNorm) {
71
+ return false;
72
+ }
73
+ if (attnOutput?.dtype !== 'f32' || (inputTensor && inputTensor.dtype !== 'f32')) {
74
+ throw new Error(
75
+ 'useSandwichRMSNormPairFusion requires f32 attention and residual tensors ' +
76
+ `(attn=${String(attnOutput?.dtype)}, residual=${String(inputTensor?.dtype)}).`
77
+ );
78
+ }
79
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
80
+ throw new Error(
81
+ `useSandwichRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
82
+ );
83
+ }
84
+ return true;
85
+ }
86
+
87
+ function releasePrecomputedInputNorm(context, recorder) {
88
+ const precomputed = context.__precomputedInputNorm ?? null;
89
+ context.__precomputedInputNorm = null;
90
+ const buffer = precomputed?.tensor?.buffer ?? null;
91
+ if (buffer) {
92
+ releaseOrTrack(recorder, buffer, context.decodeBuffers);
93
+ }
94
+ }
95
+
96
+ function takePrecomputedInputNorm(context, layerIdx, recorder) {
97
+ const precomputed = context.__precomputedInputNorm ?? null;
98
+ if (!precomputed) {
99
+ return null;
100
+ }
101
+ context.__precomputedInputNorm = null;
102
+ if (precomputed.layerIdx !== layerIdx) {
103
+ const buffer = precomputed?.tensor?.buffer ?? null;
104
+ if (buffer) {
105
+ releaseOrTrack(recorder, buffer, context.decodeBuffers);
106
+ }
107
+ throw new Error(
108
+ `Layer ${layerIdx} received stale precomputed input norm for layer ${String(precomputed.layerIdx)}.`
109
+ );
110
+ }
111
+ return precomputed.tensor;
112
+ }
113
+
114
+ function shouldUsePostFfnNextInputRMSNormPairFusion({
115
+ context,
116
+ config,
117
+ sandwichNorm,
118
+ layerIdx,
119
+ layerWeights,
120
+ nextLayerWeights,
121
+ numTokens,
122
+ hiddenSize,
123
+ activationDtype,
124
+ layerScalar,
125
+ }) {
126
+ const mergedSession = getRuntimeConfig()?.inference?.session;
127
+ if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
128
+ return false;
129
+ }
130
+ if (numTokens !== 1 || context.phase !== 'decode' || context.diffusionGemmaDecoder === true) {
131
+ return false;
132
+ }
133
+ if (context.debug === true || context.debugProbes?.length || context.operatorDiagnostics?.enabled === true) {
134
+ return false;
135
+ }
136
+ if (!context.decodeBuffers || context.pipelinePlan || hasPerLayerInputBlock(config)) {
137
+ return false;
138
+ }
139
+ if (layerScalar !== 1 || activationDtype !== 'f32') {
140
+ return false;
141
+ }
142
+ const nextLayerIdx = layerIdx + 1;
143
+ if (nextLayerIdx >= config.numLayers) {
144
+ return false;
145
+ }
146
+ const nextLayerType = config.layerTypes?.[nextLayerIdx];
147
+ if (isConvLayerType(nextLayerType) || isLinearLayerType(nextLayerType)) {
148
+ return false;
149
+ }
150
+ if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostFeedforwardNorm) {
151
+ return false;
152
+ }
153
+ if (!layerWeights?.postFeedforwardNorm || !nextLayerWeights?.inputNorm) {
154
+ return false;
155
+ }
156
+ if (config.useMoE && isMoELayer(layerIdx, config)) {
157
+ return false;
158
+ }
159
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
160
+ throw new Error(
161
+ `usePostFfnNextInputRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
162
+ );
163
+ }
164
+ return true;
165
+ }
166
+
49
167
 
50
168
  export function isMoELayer(layerIdx, config) {
51
169
  if (!config.useMoE) return false;
@@ -504,10 +622,10 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
504
622
  // Debug entry (uses debug-utils)
505
623
  logLayer(layerIdx, 'enter', isPrefill, { numTokens });
506
624
 
507
- const device = getDevice();
625
+ const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
626
+ const device = recorder?.device ?? getDevice();
508
627
  if (!device) throw new Error('No GPU device available');
509
628
 
510
- const { config, weights, weightConfig, debugFlags, kvCache, ropeFreqsCos, ropeFreqsSin, recorder } = context;
511
629
  assertSupportedLayerRuntime(layerIdx, config);
512
630
  const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
513
631
 
@@ -522,6 +640,16 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
522
640
  const sandwichNorm = detectSandwichNorm(config);
523
641
  const lastTokenIdx = Math.max(0, numTokens - 1);
524
642
 
643
+ await runProbes('layer_in', inputBuffer, {
644
+ layerIdx,
645
+ numTokens,
646
+ hiddenSize,
647
+ probes: context.debugProbes,
648
+ recorder,
649
+ operatorDiagnostics: context.operatorDiagnostics,
650
+ dtype: inputTensor.dtype,
651
+ });
652
+
525
653
  if (context.pipelinePlan) {
526
654
  return processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm);
527
655
  }
@@ -615,7 +743,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
615
743
  let attentionHeadDim = resolveAttentionHeadDim(config, layerType);
616
744
  let attentionNumKVHeads = resolveAttentionNumKVHeads(config, layerType, layerWeights, attentionHeadDim);
617
745
  let disableRoPE = false;
618
- let queryKeyNorm = config.queryKeyNorm;
746
+ let queryKeyNorm = config.queryKeyNorm === true;
747
+ const diffusionGemmaDecoder = context.diffusionGemmaDecoder === true;
748
+ if (queryKeyNorm && Array.isArray(config.queryKeyNormLayers)) {
749
+ queryKeyNorm = config.queryKeyNormLayers.includes(layerIdx);
750
+ }
619
751
  const { sharedKVSourceLayerIdx, storeSharedKV } = resolveAttentionKVSharing(config, layerIdx, layerType);
620
752
 
621
753
  const attnConfig = {
@@ -637,11 +769,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
637
769
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
638
770
  queryPreAttnScalar: config.queryPreAttnScalar,
639
771
  queryKeyNorm,
772
+ queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
640
773
  valueNorm: config.valueNorm,
641
774
  attentionOutputGate: config.attentionOutputGate,
642
775
  outputGateType: config.outputGateType ?? null,
643
- causalAttention: config.causalAttention,
644
- multimodalBidirectionalSpan: isSlidingLayerType(layerType)
776
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
777
+ multimodalBidirectionalSpan: !diffusionGemmaDecoder && isSlidingLayerType(layerType)
645
778
  ? (context.multimodalBidirectionalSpan ?? null)
646
779
  : null,
647
780
  rmsNormWeightOffset: config.rmsNormWeightOffset,
@@ -654,9 +787,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
654
787
  disableRoPE,
655
788
  sharedKVSourceLayerIdx,
656
789
  storeSharedKV,
790
+ diffusionGemmaDecoder,
657
791
  };
658
792
 
659
793
  validateAttnConfig(attnConfig, `L${layerIdx}`);
794
+ attnConfig.precomputedInputNorm = takePrecomputedInputNorm(context, layerIdx, recorder);
660
795
 
661
796
  const attnState = {
662
797
  ropeFreqsCos: (isLocalLayer && context.ropeLocalCos)
@@ -738,9 +873,37 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
738
873
 
739
874
  // 2. Handle residual connection based on architecture
740
875
 
876
+ let precomputedFfnInput = null;
741
877
  if (residualFused) {
742
878
  postAttn = attnOutput;
743
- if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
879
+ if (shouldUseSandwichRMSNormPairFusion({
880
+ context,
881
+ sandwichNorm,
882
+ layerWeights,
883
+ numTokens,
884
+ hiddenSize,
885
+ attnOutput,
886
+ inputTensor: null,
887
+ })) {
888
+ const postNormWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
889
+ const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
890
+ const pair = await doSandwichRMSNormPair(attnOutput, null, postNormWeightBuf, preNormWeightBuf, rmsNormEps, {
891
+ batchSize: numTokens,
892
+ hiddenSize,
893
+ label: `L${layerIdx}.post_attn_pre_ffn_norm`,
894
+ layerIdx,
895
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
896
+ }, recorder);
897
+ postAttn = pair.postAttn;
898
+ precomputedFfnInput = pair.ffnInput;
899
+ if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, postNormWeightBuf);
900
+ if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
901
+ if (recorder) {
902
+ recorder.trackTemporaryBuffer(attnOutput.buffer);
903
+ } else {
904
+ releaseBuffer(attnOutput.buffer);
905
+ }
906
+ } else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
744
907
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
745
908
  postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
746
909
  batchSize: numTokens,
@@ -758,21 +921,52 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
758
921
  }
759
922
  } else if (sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm && layerWeights?.postAttentionNorm) {
760
923
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttentionNorm, 'post_attention_norm', weightConfig, debugFlags);
761
- const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
762
- batchSize: numTokens,
924
+ if (shouldUseSandwichRMSNormPairFusion({
925
+ context,
926
+ sandwichNorm,
927
+ layerWeights,
928
+ numTokens,
763
929
  hiddenSize,
764
- label: `L${layerIdx}.post_attn_norm`,
765
- layerIdx,
766
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
767
- }, recorder);
768
- postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
769
- label: `L${layerIdx}.post_attn_residual`,
770
- layerIdx,
771
- executionPolicies: context.executionPolicies ?? null,
772
- });
930
+ attnOutput,
931
+ inputTensor,
932
+ })) {
933
+ const preNormWeightBuf = getNormWeightBuffer(layerWeights.preFeedforwardNorm, 'pre_feedforward_norm', weightConfig, debugFlags);
934
+ const pair = await doSandwichRMSNormPair(attnOutput, inputTensor, normWeightBuf, preNormWeightBuf, rmsNormEps, {
935
+ batchSize: numTokens,
936
+ hiddenSize,
937
+ label: `L${layerIdx}.post_attn_pre_ffn_norm`,
938
+ layerIdx,
939
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
940
+ }, recorder);
941
+ postAttn = pair.postAttn;
942
+ precomputedFfnInput = pair.ffnInput;
943
+ if (!isGpuBufferInstance(layerWeights.preFeedforwardNorm) && !isWeightBuffer(layerWeights.preFeedforwardNorm)) releaseOrTrack(recorder, preNormWeightBuf);
944
+ } else if (attnOutput.dtype === inputTensor.dtype) {
945
+ postAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
946
+ batchSize: numTokens,
947
+ hiddenSize,
948
+ residual: inputTensor,
949
+ label: `L${layerIdx}.post_attn_norm`,
950
+ layerIdx,
951
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
952
+ }, recorder);
953
+ } else {
954
+ const normalizedAttn = await doRMSNorm(attnOutput, normWeightBuf, rmsNormEps, {
955
+ batchSize: numTokens,
956
+ hiddenSize,
957
+ label: `L${layerIdx}.post_attn_norm`,
958
+ layerIdx,
959
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
960
+ }, recorder);
961
+ postAttn = await doResidualAdd(normalizedAttn, inputTensor, size, recorder, {
962
+ label: `L${layerIdx}.post_attn_residual`,
963
+ layerIdx,
964
+ executionPolicies: context.executionPolicies ?? null,
965
+ });
966
+ releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
967
+ }
773
968
 
774
969
  if (!isGpuBufferInstance(layerWeights.postAttentionNorm) && !isWeightBuffer(layerWeights.postAttentionNorm)) releaseOrTrack(recorder, normWeightBuf);
775
- releaseOrTrack(recorder, normalizedAttn.buffer, context.decodeBuffers);
776
970
  if (recorder) {
777
971
  recorder.trackTemporaryBuffer(attnOutput.buffer);
778
972
  } else {
@@ -819,10 +1013,60 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
819
1013
  // 3. Feed-forward network
820
1014
 
821
1015
  let outputTensor;
1016
+ const layerScalar = resolveLayerScalarValue(layerWeights?.layerScalar ?? null);
1017
+ const requestFfnLayerScalarFusion = layerScalar !== 1
1018
+ && !hasPerLayerInputBlock(config);
1019
+ const nextLayerIdx = layerIdx + 1;
1020
+ const nextLayerWeights = nextLayerIdx < config.numLayers
1021
+ ? weights.get(`layer_${nextLayerIdx}`)
1022
+ : null;
1023
+ const usePostFfnNextInputNormPair = shouldUsePostFfnNextInputRMSNormPairFusion({
1024
+ context,
1025
+ config,
1026
+ sandwichNorm,
1027
+ layerIdx,
1028
+ layerWeights,
1029
+ nextLayerWeights,
1030
+ numTokens,
1031
+ hiddenSize,
1032
+ activationDtype,
1033
+ layerScalar,
1034
+ });
1035
+ let layerScalarFused = false;
822
1036
  if (sandwichNorm.useSandwichNorm) {
823
- outputTensor = await processFFNWithSandwichNorm(layerIdx, postAttn, numTokens, size, context, layerWeights, sandwichNorm);
1037
+ context.__postFfnNextInputNorm = usePostFfnNextInputNormPair
1038
+ ? { layerIdx: nextLayerIdx, weight: nextLayerWeights.inputNorm }
1039
+ : null;
1040
+ try {
1041
+ outputTensor = await processFFNWithSandwichNorm(
1042
+ layerIdx,
1043
+ postAttn,
1044
+ numTokens,
1045
+ size,
1046
+ context,
1047
+ layerWeights,
1048
+ sandwichNorm,
1049
+ requestFfnLayerScalarFusion ? layerScalar : 1,
1050
+ precomputedFfnInput
1051
+ );
1052
+ } finally {
1053
+ context.__postFfnNextInputNorm = null;
1054
+ }
1055
+ layerScalarFused = context.__layerScalarFusedFired === true;
1056
+ context.__layerScalarFusedFired = false;
824
1057
  } else {
825
- outputTensor = await processFFNStandard(layerIdx, postAttn, numTokens, size, context, layerWeights, fusedResidualForFFN);
1058
+ outputTensor = await processFFNStandard(
1059
+ layerIdx,
1060
+ postAttn,
1061
+ numTokens,
1062
+ size,
1063
+ context,
1064
+ layerWeights,
1065
+ fusedResidualForFFN,
1066
+ requestFfnLayerScalarFusion ? layerScalar : 1
1067
+ );
1068
+ layerScalarFused = context.__layerScalarFusedFired === true;
1069
+ context.__layerScalarFusedFired = false;
826
1070
  }
827
1071
 
828
1072
  // Keep activation dtype consistent across layers. Some FFN paths can emit f32
@@ -862,10 +1106,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
862
1106
  }
863
1107
 
864
1108
  let finalOutput = outputTensor;
865
- const scaledOutput = await applyLayerScalar(layerIdx, finalOutput, size, context, layerWeights);
866
- if (scaledOutput.buffer !== finalOutput.buffer) {
867
- releaseOrTrack(recorder, finalOutput.buffer, context.decodeBuffers);
868
- finalOutput = scaledOutput;
1109
+ if (!layerScalarFused) {
1110
+ const scaledOutput = await applyLayerScalar(layerIdx, finalOutput, size, context, layerWeights);
1111
+ if (scaledOutput.buffer !== finalOutput.buffer) {
1112
+ releaseOrTrack(recorder, finalOutput.buffer, context.decodeBuffers);
1113
+ finalOutput = scaledOutput;
1114
+ }
869
1115
  }
870
1116
  await debugLayerTensor(context, layerIdx, 'final layer output', finalOutput, numTokens, hiddenSize);
871
1117
  await runProbes('layer_out', finalOutput.buffer, {
@@ -904,6 +1150,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
904
1150
  released.add(buf);
905
1151
  releaseOrTrack(recorder, buf);
906
1152
  };
1153
+ releasePrecomputedInputNorm(context, recorder);
907
1154
  if (postAttn?.buffer) releaseOnce(postAttn.buffer);
908
1155
  if (attnOutput?.buffer && attnOutput.buffer !== postAttn?.buffer) releaseOnce(attnOutput.buffer);
909
1156
  throw error;