doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -1,4 +1,5 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
+ import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
2
3
  import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
3
4
  import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
4
5
  import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
@@ -24,7 +25,7 @@ function kernelPathVariantRequiresQ4KWeights(variant) {
24
25
  return shaderFile.startsWith('fused_matmul_q4');
25
26
  }
26
27
 
27
- function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
28
+ function qkvProjectionRequiresQ4KWeights(layerIdx, kernelPath) {
28
29
  if (!kernelPath) {
29
30
  return false;
30
31
  }
@@ -36,7 +37,82 @@ function shouldSkipQKVFusionForLayer(layerIdx, kernelPath) {
36
37
  return kernelPathVariantRequiresQ4KWeights(decodeVariant);
37
38
  }
38
39
 
39
- export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
40
+ function normalizeProjectionDtype(weight) {
41
+ const dtype = typeof weight?.dtype === 'string'
42
+ ? weight.dtype.toLowerCase()
43
+ : null;
44
+ if (dtype === 'bf16') return 'f16';
45
+ if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
46
+ return null;
47
+ }
48
+
49
+ function resolveProjectionStorageFormat(weight, expectedRows, hiddenSize) {
50
+ const dtype = normalizeProjectionDtype(weight);
51
+ if (dtype === 'q4k') {
52
+ const layout = String(weight?.layout ?? 'row').toLowerCase();
53
+ if (layout !== 'row') {
54
+ return null;
55
+ }
56
+ const bytesPerRow = q4kBlockCount(hiddenSize) * Q4K_BLOCK_BYTES;
57
+ const byteLength = expectedRows * bytesPerRow;
58
+ if (weight?.buffer?.size < byteLength) {
59
+ return null;
60
+ }
61
+ return {
62
+ dtype,
63
+ layout,
64
+ byteLength,
65
+ bytesPerElement: null,
66
+ bytesPerRow,
67
+ };
68
+ }
69
+
70
+ if (dtype === 'f16' || dtype === 'f32') {
71
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
72
+ const byteLength = expectedRows * hiddenSize * bytesPerElement;
73
+ if (weight?.buffer?.size < byteLength) {
74
+ return null;
75
+ }
76
+ return {
77
+ dtype,
78
+ layout: weight?.layout ?? null,
79
+ byteLength,
80
+ bytesPerElement,
81
+ bytesPerRow: hiddenSize * bytesPerElement,
82
+ };
83
+ }
84
+
85
+ const minF16Bytes = expectedRows * hiddenSize * 2;
86
+ const minF32Bytes = expectedRows * hiddenSize * 4;
87
+ if (weight?.buffer?.size >= minF32Bytes) {
88
+ return {
89
+ dtype: 'f32',
90
+ layout: weight?.layout ?? null,
91
+ byteLength: minF32Bytes,
92
+ bytesPerElement: 4,
93
+ bytesPerRow: hiddenSize * 4,
94
+ };
95
+ }
96
+ if (weight?.buffer?.size >= minF16Bytes) {
97
+ return {
98
+ dtype: 'f16',
99
+ layout: weight?.layout ?? null,
100
+ byteLength: minF16Bytes,
101
+ bytesPerElement: 2,
102
+ bytesPerRow: hiddenSize * 2,
103
+ };
104
+ }
105
+ return null;
106
+ }
107
+
108
+ function formatsMatch(a, b) {
109
+ return a?.dtype === b?.dtype
110
+ && a?.layout === b?.layout
111
+ && a?.bytesPerElement === b?.bytesPerElement
112
+ && a?.bytesPerRow === b?.bytesPerRow;
113
+ }
114
+
115
+ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
40
116
  const device = getDevice();
41
117
  if (!device) {
42
118
  log.debug('QKV Fusion', 'No GPU device, skipping fusion');
@@ -52,12 +128,14 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
52
128
  );
53
129
  return;
54
130
  }
55
-
56
131
  const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
57
132
  const qSize = numHeads * headDim;
133
+ const hasAttentionOutputGate = modelConfig?.attentionOutputGate === true;
134
+ const qProjSize = hasAttentionOutputGate ? qSize * 2 : qSize;
58
135
  const kSize = numKVHeads * headDim;
59
136
  const vSize = numKVHeads * headDim;
60
137
  const qkvSize = qSize + kSize + vSize;
138
+ const allowQ4K = options.allowQ4K === true;
61
139
 
62
140
  const resolveWeight = (value) => {
63
141
  if (isWeightBuffer(value)) {
@@ -81,19 +159,6 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
81
159
 
82
160
  log.debug('QKV Fusion', `Fusing Q/K/V weights for ${numLayers} layers (${qSize}+${kSize}+${vSize}=${qkvSize})`);
83
161
 
84
- const resolveBytesPerElement = (weight, expectedElements) => {
85
- const dtype = typeof weight?.dtype === 'string'
86
- ? weight.dtype.toLowerCase()
87
- : null;
88
- if (dtype === 'f16' || dtype === 'bf16') return 2;
89
- if (dtype === 'f32') return 4;
90
- const minF16Bytes = expectedElements * 2;
91
- const minF32Bytes = expectedElements * 4;
92
- if (weight?.buffer?.size >= minF32Bytes) return 4;
93
- if (weight?.buffer?.size >= minF16Bytes) return 2;
94
- return 0;
95
- };
96
-
97
162
  let fusedCount = 0;
98
163
  for (let l = 0; l < numLayers; l++) {
99
164
  const weights = layerWeights.get(`layer_${l}`);
@@ -101,13 +166,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
101
166
 
102
167
  // Skip if already fused or if weights are not GPUBuffers
103
168
  if (weights.qkvProj) continue;
104
- if (shouldSkipQKVFusionForLayer(l, kernelPath)) {
105
- log.debug(
106
- 'QKV Fusion',
107
- `Layer ${l}: skipped because active kernel path requires Q4K weights for qkv_proj`
108
- );
109
- continue;
110
- }
169
+ const requiresQ4K = qkvProjectionRequiresQ4KWeights(l, kernelPath);
111
170
  const qProj = resolveWeight(weights.qProj);
112
171
  const kProj = resolveWeight(weights.kProj);
113
172
  const vProj = resolveWeight(weights.vProj);
@@ -115,72 +174,97 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
115
174
  continue;
116
175
  }
117
176
 
118
- const qExpectedElements = qSize * hiddenSize;
119
- const kExpectedElements = kSize * hiddenSize;
120
- const vExpectedElements = vSize * hiddenSize;
121
- const bytesPerElement = resolveBytesPerElement(qProj, qExpectedElements);
122
- const kBytesPerElement = resolveBytesPerElement(kProj, kExpectedElements);
123
- const vBytesPerElement = resolveBytesPerElement(vProj, vExpectedElements);
177
+ const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
178
+ const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
179
+ const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
124
180
 
125
181
  // Pool allocation can round GPUBuffer.size up, so infer logical dtype first and
126
182
  // only use buffer size as a minimum-size inference.
127
- if ((bytesPerElement !== 2 && bytesPerElement !== 4)
128
- || kBytesPerElement !== bytesPerElement
129
- || vBytesPerElement !== bytesPerElement) {
183
+ if (!qFormat || !formatsMatch(qFormat, kFormat) || !formatsMatch(qFormat, vFormat)) {
130
184
  log.debug(
131
185
  'QKV Fusion',
132
- `Layer ${l}: inconsistent projection dtypes (q=${bytesPerElement}, k=${kBytesPerElement}, v=${vBytesPerElement}), skipping`
186
+ `Layer ${l}: inconsistent projection storage formats, skipping`
133
187
  );
134
188
  continue;
135
189
  }
136
190
 
137
- const normalizedDtype = typeof qProj.dtype === 'string'
138
- ? qProj.dtype.toLowerCase()
139
- : null;
140
- const dtype = normalizedDtype === 'bf16'
141
- ? 'f16'
142
- : (
143
- normalizedDtype === 'f16' || normalizedDtype === 'f32'
144
- ? normalizedDtype
145
- : selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement })
146
- );
191
+ if (requiresQ4K && qFormat.dtype !== 'q4k') {
192
+ log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} pack`);
193
+ continue;
194
+ }
195
+ if (qFormat.dtype === 'q4k' && !allowQ4K) {
196
+ log.debug('QKV Fusion', `Layer ${l}: Q4K QKV fusion requires explicit runtime opt-in`);
197
+ continue;
198
+ }
199
+
200
+ const dtype = qFormat.dtype ?? selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', {
201
+ bytesPerElement: qFormat.bytesPerElement,
202
+ });
147
203
  const layout = qProj.layout ?? kProj.layout ?? vProj.layout ?? 'row';
148
204
  let fusedShape = [qkvSize, hiddenSize];
149
- if (Array.isArray(qProj.shape) && qProj.shape.length === 2) {
150
- if (qProj.shape[0] === qSize && qProj.shape[1] === hiddenSize) {
205
+ if (qFormat.dtype !== 'q4k' && Array.isArray(qProj.shape) && qProj.shape.length === 2) {
206
+ if (qProj.shape[0] === qProjSize && qProj.shape[1] === hiddenSize) {
151
207
  fusedShape = [qkvSize, hiddenSize];
152
- } else if (qProj.shape[1] === qSize && qProj.shape[0] === hiddenSize) {
208
+ } else if (qProj.shape[1] === qProjSize && qProj.shape[0] === hiddenSize) {
153
209
  fusedShape = [hiddenSize, qkvSize];
154
210
  }
155
211
  }
156
212
 
157
- // Create fused QKV buffer: [qkvSize, hiddenSize] row-major
158
- // Each row is concatenated: [q_row, k_row, v_row]
213
+ const qBytes = qSize * qFormat.bytesPerRow;
214
+ const qGateBytes = hasAttentionOutputGate ? qBytes : 0;
215
+
216
+ // Create fused QKV buffer: [qkvSize, hiddenSize] row-major.
217
+ // attentionOutputGate models keep Q rows in qkvProj and gate rows in qGateProj.
159
218
  const qkvBuffer = device.createBuffer({
160
219
  label: `layer_${l}_qkv_proj`,
161
- size: qkvSize * hiddenSize * bytesPerElement,
220
+ size: qBytes + kFormat.byteLength + vFormat.byteLength,
162
221
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
163
222
  });
223
+ const qGateBuffer = hasAttentionOutputGate
224
+ ? device.createBuffer({
225
+ label: `layer_${l}_q_gate_proj`,
226
+ size: qGateBytes,
227
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
228
+ })
229
+ : null;
164
230
 
165
- // Copy Q, K, V weights into fused buffer
166
- // Q: [qSize, hiddenSize] -> offset 0
167
- // K: [kSize, hiddenSize] -> offset qSize * hiddenSize * bytesPerElement
168
- // V: [vSize, hiddenSize] -> offset (qSize + kSize) * hiddenSize * bytesPerElement
169
231
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
170
- encoder.copyBufferToBuffer(
171
- qProj.buffer, 0,
172
- qkvBuffer, 0,
173
- qSize * hiddenSize * bytesPerElement
174
- );
232
+ if (hasAttentionOutputGate) {
233
+ for (let head = 0; head < numHeads; head++) {
234
+ const srcHeadOffset = head * headDim * 2 * qFormat.bytesPerRow;
235
+ const dstHeadOffset = head * headDim * qFormat.bytesPerRow;
236
+ const headBytes = headDim * qFormat.bytesPerRow;
237
+ encoder.copyBufferToBuffer(
238
+ qProj.buffer,
239
+ srcHeadOffset,
240
+ qkvBuffer,
241
+ dstHeadOffset,
242
+ headBytes
243
+ );
244
+ encoder.copyBufferToBuffer(
245
+ qProj.buffer,
246
+ srcHeadOffset + headBytes,
247
+ qGateBuffer,
248
+ dstHeadOffset,
249
+ headBytes
250
+ );
251
+ }
252
+ } else {
253
+ encoder.copyBufferToBuffer(
254
+ qProj.buffer, 0,
255
+ qkvBuffer, 0,
256
+ qBytes
257
+ );
258
+ }
175
259
  encoder.copyBufferToBuffer(
176
260
  kProj.buffer, 0,
177
- qkvBuffer, qSize * hiddenSize * bytesPerElement,
178
- kSize * hiddenSize * bytesPerElement
261
+ qkvBuffer, qBytes,
262
+ kFormat.byteLength
179
263
  );
180
264
  encoder.copyBufferToBuffer(
181
265
  vProj.buffer, 0,
182
- qkvBuffer, (qSize + kSize) * hiddenSize * bytesPerElement,
183
- vSize * hiddenSize * bytesPerElement
266
+ qkvBuffer, qBytes + kFormat.byteLength,
267
+ vFormat.byteLength
184
268
  );
185
269
  device.queue.submit([encoder.finish()]);
186
270
 
@@ -194,6 +278,15 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null) {
194
278
  );
195
279
  weights.qkvSizes = [qSize, kSize, vSize];
196
280
  weights.qkvDtype = dtype;
281
+ if (qGateBuffer) {
282
+ weights.qGateProj = createWeightBuffer(
283
+ qGateBuffer,
284
+ dtype,
285
+ layout,
286
+ [qSize, hiddenSize],
287
+ `layer_${l}_q_gate_proj`
288
+ );
289
+ }
197
290
  fusedCount++;
198
291
  }
199
292
 
@@ -44,6 +44,7 @@ import type {
44
44
  import type { LoaderDebugConfigSchema } from '../../../config/schema/debug.schema.js';
45
45
  import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
46
46
  import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
47
+ import type { DopplerLoader } from '../../../loader/doppler-loader.js';
47
48
 
48
49
  export interface PipelineStorageContext {
49
50
  preflight?: () => Promise<void>;
@@ -67,6 +68,15 @@ export interface PipelineStorageContext {
67
68
  close?: () => Promise<void>;
68
69
  }
69
70
 
71
+ export interface EmulationContext {
72
+ config: {
73
+ topology: { gpuCount: number };
74
+ timingMode: string;
75
+ };
76
+ getStats(): unknown;
77
+ destroy(): void | Promise<void>;
78
+ }
79
+
70
80
  export function createNodeFileShardStorageContext(
71
81
  baseUrl: string | null | undefined,
72
82
  manifest: Manifest
@@ -90,6 +100,10 @@ export interface PipelineContexts {
90
100
  runtimeConfig?: Partial<RuntimeConfigSchema> | RuntimeConfigSchema;
91
101
  /** Progress callback for weight loading */
92
102
  onProgress?: (progress: { percent: number; message?: string }) => void;
103
+ /** Optional caller-owned loader instance for isolated model residency */
104
+ loader?: DopplerLoader;
105
+ /** True when the pipeline should release the injected loader on unload */
106
+ ownsLoader?: boolean;
93
107
  }
94
108
 
95
109
  /**
@@ -219,6 +233,7 @@ export function initTokenizer(
219
233
  * Weight loading result.
220
234
  */
221
235
  export interface WeightLoadResult {
236
+ loader: DopplerLoader;
222
237
  layerWeights: Map<string, LayerWeights>;
223
238
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
224
239
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
@@ -250,6 +265,7 @@ export interface LoadWeightsOptions {
250
265
  keepF32Weights?: boolean;
251
266
  loaderDebug?: LoaderDebugConfigSchema | null;
252
267
  perLayerInputSession?: ExecutionV1PerLayerInputsSessionSchema | null;
268
+ loader?: DopplerLoader;
253
269
  }
254
270
 
255
271
  /**
@@ -334,7 +350,10 @@ export function initSpeculativeDecoder(
334
350
  export function fuseQKVWeights(
335
351
  layerWeights: Map<string, LayerWeights>,
336
352
  modelConfig: ParsedModelConfig,
337
- kernelPath?: KernelPathSchema | null
353
+ kernelPath?: KernelPathSchema | null,
354
+ options?: {
355
+ allowQ4K?: boolean;
356
+ }
338
357
  ): void;
339
358
 
340
359
  /**
@@ -349,7 +368,7 @@ export function fuseQKVWeights(
349
368
  */
350
369
  export function initEmulation(
351
370
  runtimeConfig: RuntimeConfigSchema
352
- ): Promise<import('/proto/simulator/index.js').EmulationContext | null>;
371
+ ): Promise<EmulationContext | null>;
353
372
 
354
373
  /**
355
374
  * Destroy emulation context and clean up resources.
@@ -357,5 +376,5 @@ export function initEmulation(
357
376
  * @param emulation - Emulation context to destroy
358
377
  */
359
378
  export function destroyEmulation(
360
- emulation: import('/proto/simulator/index.js').EmulationContext | null
379
+ emulation: EmulationContext | null
361
380
  ): Promise<void>;
@@ -879,7 +879,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
879
879
  throw new Error('runtime.loading.shardCache.verifyHashes is required.');
880
880
  }
881
881
 
882
- const dopplerLoader = getDopplerLoader(loadingConfig);
882
+ const dopplerLoader = options.loader ?? getDopplerLoader(loadingConfig);
883
883
  const keepF32Weights = options.keepF32Weights === true;
884
884
  dopplerLoader.setQ4KConfig(
885
885
  resolveQ4KConfig(
@@ -997,7 +997,6 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
997
997
  layerWeights.set(`layer_${l}`, weights);
998
998
  }
999
999
  }
1000
-
1001
1000
  // Collect per-layer router weights for MoE
1002
1001
 
1003
1002
  const layerRouterWeights = new Map();
@@ -1017,6 +1016,7 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1017
1016
  }
1018
1017
 
1019
1018
  return {
1019
+ loader: dopplerLoader,
1020
1020
  layerWeights,
1021
1021
  embeddings: dopplerLoader.embeddings,
1022
1022
  lmHead: dopplerLoader.lmHead,
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { log, trace } from '../../../debug/index.js';
4
4
  import { getRuntimeConfig } from '../../../config/runtime.js';
5
- import { getDevice } from '../../../gpu/device.js';
5
+ import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
6
6
  import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
7
7
  import { allowReadback } from '../../../gpu/perf-guards.js';
8
8
  import { createTensor } from '../../../gpu/tensor.js';
@@ -144,7 +144,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
144
144
  return false;
145
145
  }
146
146
  const nextLayerType = config.layerTypes?.[nextLayerIdx];
147
- if (isConvLayerType(nextLayerType) || isLinearLayerType(nextLayerType)) {
147
+ if (isConvLayerType(nextLayerType)) {
148
148
  return false;
149
149
  }
150
150
  if (!sandwichNorm.useSandwichNorm || !sandwichNorm.hasPostFeedforwardNorm) {
@@ -164,6 +164,56 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
164
164
  return true;
165
165
  }
166
166
 
167
+ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
168
+ context,
169
+ config,
170
+ sandwichNorm,
171
+ layerIdx,
172
+ nextLayerWeights,
173
+ numTokens,
174
+ hiddenSize,
175
+ activationDtype,
176
+ layerScalar,
177
+ residualBranchScale,
178
+ }) {
179
+ const mergedSession = getRuntimeConfig()?.inference?.session;
180
+ if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
181
+ return false;
182
+ }
183
+ if (numTokens !== 1 || context.phase !== 'decode' || context.diffusionGemmaDecoder === true) {
184
+ return false;
185
+ }
186
+ if (context.debug === true || context.debugProbes?.length || context.operatorDiagnostics?.enabled === true) {
187
+ return false;
188
+ }
189
+ if (!context.decodeBuffers || context.pipelinePlan || hasPerLayerInputBlock(config)) {
190
+ return false;
191
+ }
192
+ if (sandwichNorm.useSandwichNorm || layerScalar !== 1 || residualBranchScale !== 1 || activationDtype !== 'f32') {
193
+ return false;
194
+ }
195
+ if (getKernelCapabilities()?.hasSubgroups !== true) {
196
+ return false;
197
+ }
198
+ const nextLayerIdx = layerIdx + 1;
199
+ if (nextLayerIdx >= config.numLayers) {
200
+ return false;
201
+ }
202
+ const nextLayerType = config.layerTypes?.[nextLayerIdx];
203
+ if (isConvLayerType(nextLayerType)) {
204
+ return false;
205
+ }
206
+ if (!nextLayerWeights?.inputNorm) {
207
+ return false;
208
+ }
209
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0 || hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
210
+ throw new Error(
211
+ `usePostFfnNextInputRMSNormPairFusion requires hiddenSize in 1..${RMSNORM_PAIR_CACHE_LIMIT}; got ${String(hiddenSize)}.`
212
+ );
213
+ }
214
+ return true;
215
+ }
216
+
167
217
 
168
218
  export function isMoELayer(layerIdx, config) {
169
219
  if (!config.useMoE) return false;
@@ -395,6 +445,7 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
395
445
  let activatedTensor = null;
396
446
  let projectedTensor = null;
397
447
  let normalizedTensor = null;
448
+ let residualAddTensor = null;
398
449
  let outputTensor = null;
399
450
 
400
451
  try {
@@ -501,11 +552,19 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
501
552
  releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
502
553
  projectedTensor = null;
503
554
 
504
- outputTensor = await doResidualAdd(normalizedTensor, residualTensor, size, recorder, {
555
+ residualAddTensor = normalizedTensor;
556
+ if (residualAddTensor.dtype !== residualTensor.dtype) {
557
+ residualAddTensor = await doCast(residualAddTensor, residualTensor.dtype, recorder);
558
+ }
559
+ outputTensor = await doResidualAdd(residualAddTensor, residualTensor, size, recorder, {
505
560
  label: `L${layerIdx}.per_layer_input_residual`,
506
561
  layerIdx,
507
562
  executionPolicies: context.executionPolicies ?? null,
508
563
  });
564
+ if (residualAddTensor.buffer !== normalizedTensor.buffer) {
565
+ releaseOrTrack(recorder, residualAddTensor.buffer, decodeBuffers);
566
+ }
567
+ residualAddTensor = null;
509
568
  releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
510
569
  normalizedTensor = null;
511
570
 
@@ -523,6 +582,9 @@ export async function applyPerLayerInputBlock(layerIdx, hiddenTensor, numTokens,
523
582
  return outputTensor;
524
583
  } catch (error) {
525
584
  if (outputTensor?.buffer) releaseOrTrack(recorder, outputTensor.buffer, decodeBuffers);
585
+ if (residualAddTensor?.buffer && residualAddTensor.buffer !== normalizedTensor?.buffer) {
586
+ releaseOrTrack(recorder, residualAddTensor.buffer, decodeBuffers);
587
+ }
526
588
  if (normalizedTensor?.buffer) releaseOrTrack(recorder, normalizedTensor.buffer, decodeBuffers);
527
589
  if (projectedTensor?.buffer) releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
528
590
  if (activatedTensor?.buffer) releaseOrTrack(recorder, activatedTensor.buffer, decodeBuffers);
@@ -753,6 +815,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
753
815
  linearRuntime: context.linearAttentionRuntime ?? null,
754
816
  getWeightBuffer: (weight, label) => getWeightBuffer(weight, label),
755
817
  getNormWeightBuffer: (weight, label) => getNormWeightBuffer(weight, label, weightConfig, debugFlags),
818
+ precomputedInputNorm: takePrecomputedInputNorm(context, layerIdx, recorder),
756
819
  debugProbes: context.debugProbes,
757
820
  operatorDiagnostics: context.operatorDiagnostics,
758
821
  recorder: recorder ?? null,
@@ -826,6 +889,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
826
889
  operatorDiagnostics: context.operatorDiagnostics,
827
890
  linearRuntime: context.linearAttentionRuntime ?? null,
828
891
  executionPolicies: context.executionPolicies ?? null,
892
+ skipKVCacheWrites: context.skipKVCacheWrites === true,
829
893
  };
830
894
 
831
895
  const attnResult = await doAttention(
@@ -1058,6 +1122,18 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
1058
1122
  activationDtype,
1059
1123
  layerScalar,
1060
1124
  });
1125
+ const useStandardPostFfnNextInputNormPair = shouldUseStandardPostFfnNextInputRMSNormPairFusion({
1126
+ context,
1127
+ config,
1128
+ sandwichNorm,
1129
+ layerIdx,
1130
+ nextLayerWeights,
1131
+ numTokens,
1132
+ hiddenSize,
1133
+ activationDtype,
1134
+ layerScalar,
1135
+ residualBranchScale,
1136
+ });
1061
1137
  let layerScalarFused = false;
1062
1138
  if (sandwichNorm.useSandwichNorm) {
1063
1139
  context.__postFfnNextInputNorm = usePostFfnNextInputNormPair
@@ -1087,11 +1163,14 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
1087
1163
  numTokens,
1088
1164
  size,
1089
1165
  context,
1090
- layerWeights,
1091
- fusedResidualForFFN,
1092
- requestFfnLayerScalarFusion ? layerScalar : 1,
1093
- residualBranchScale
1094
- );
1166
+ layerWeights,
1167
+ fusedResidualForFFN,
1168
+ requestFfnLayerScalarFusion ? layerScalar : 1,
1169
+ residualBranchScale,
1170
+ useStandardPostFfnNextInputNormPair
1171
+ ? { layerIdx: nextLayerIdx, weight: nextLayerWeights.inputNorm }
1172
+ : null
1173
+ );
1095
1174
  layerScalarFused = context.__layerScalarFusedFired === true;
1096
1175
  context.__layerScalarFusedFired = false;
1097
1176
  }