doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -29,6 +29,13 @@ function assertOptionalPositiveInt(value, label) {
29
29
  return value;
30
30
  }
31
31
 
32
+ function assertOptionalNullablePositiveInt(value, label) {
33
+ if (value === undefined || value === null) {
34
+ return value;
35
+ }
36
+ return assertOptionalPositiveInt(value, label);
37
+ }
38
+
32
39
  function assertRequiredPositiveInt(value, label) {
33
40
  if (!Number.isInteger(value) || value < 1) {
34
41
  throw new Error(`[ExecutionPlan] ${label} must be a positive integer; got ${JSON.stringify(value)}.`);
@@ -139,6 +146,7 @@ function createStaticExecutionPlan({
139
146
  defaultMaxTokens: batchingConfig.maxTokens,
140
147
  readbackInterval: batchingConfig.readbackInterval,
141
148
  readbackMode: batchingConfig.readbackMode,
149
+ maxBatchDecodeTokens: batchingConfig.maxBatchDecodeTokens,
142
150
  ringTokens: batchingConfig.ringTokens,
143
151
  ringStop: batchingConfig.ringStop,
144
152
  ringStaging: batchingConfig.ringStaging,
@@ -224,6 +232,10 @@ export function compileExecutionPlanState(options) {
224
232
  maxTokens: assertRequiredPositiveInt(generationConfig.maxTokens, 'runtime.inference.generation.maxTokens'),
225
233
  readbackInterval: decodeLoopConfig.readbackInterval,
226
234
  readbackMode: assertReadbackMode(decodeLoopConfig.readbackMode),
235
+ maxBatchDecodeTokens: assertOptionalNullablePositiveInt(
236
+ decodeLoopConfig.maxBatchDecodeTokens,
237
+ 'runtimeConfig.inference.session.decodeLoop.maxBatchDecodeTokens'
238
+ ),
227
239
  ringTokens: decodeLoopConfig.ringTokens,
228
240
  ringStop: decodeLoopConfig.ringStop,
229
241
  ringStaging: decodeLoopConfig.ringStaging,
@@ -406,6 +418,7 @@ export function resolveExecutionSessionPlan(container, options = {}) {
406
418
  maxTokens: overrides.maxTokens ?? activePlan.defaultMaxTokens,
407
419
  readbackInterval: overrides.readbackInterval ?? activePlan.readbackInterval,
408
420
  readbackMode: activePlan.readbackMode,
421
+ maxBatchDecodeTokens: activePlan.maxBatchDecodeTokens,
409
422
  ringTokens: overrides.ringTokens ?? activePlan.ringTokens,
410
423
  ringStop: overrides.ringStop ?? activePlan.ringStop,
411
424
  ringStaging: overrides.ringStaging ?? activePlan.ringStaging,
@@ -434,6 +447,18 @@ export function isBatchDecodeEnabled(config) {
434
447
  }
435
448
 
436
449
  export function resolveMaxBatchDecodeTokens(config) {
450
+ if (config.configuredMaxBatchDecodeTokens !== undefined) {
451
+ const value = config.configuredMaxBatchDecodeTokens;
452
+ if (value === null) {
453
+ return null;
454
+ }
455
+ if (!Number.isInteger(value) || value < 1) {
456
+ throw new Error(
457
+ `[ExecutionPlan] configured maxBatchDecodeTokens must be null or a positive integer; got ${JSON.stringify(value)}.`
458
+ );
459
+ }
460
+ return value;
461
+ }
437
462
  const value = selectRuleValue('inference', 'execution', 'maxBatchDecodeTokens', {
438
463
  hasHotVocabularyBatchDecode: config.hasHotVocabularyBatchDecode === true,
439
464
  hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
@@ -457,6 +482,18 @@ export function resolveMaxBatchDecodeTokens(config) {
457
482
  }
458
483
 
459
484
  export function resolvePrefillRecorderChunkLayers(config) {
485
+ if (config.configuredPrefillChunkLayers !== undefined) {
486
+ const configured = config.configuredPrefillChunkLayers;
487
+ if (configured === null) {
488
+ throw new Error('[ExecutionPlan] configured prefillChunkLayers cannot be null.');
489
+ }
490
+ if (!Number.isInteger(configured) || configured < 1) {
491
+ throw new Error(
492
+ `[ExecutionPlan] configured prefillChunkLayers must be a positive integer; got ${JSON.stringify(configured)}.`
493
+ );
494
+ }
495
+ return configured;
496
+ }
460
497
  const value = selectRuleValue('inference', 'execution', 'prefillRecorderChunkLayers', {
461
498
  hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
462
499
  numTokens: config.numTokens,
@@ -21,6 +21,12 @@ export interface ExecutionV1LaneIntegrity {
21
21
  };
22
22
  status: 'matches' | 'transformed';
23
23
  transforms: string[];
24
+ policy?: {
25
+ kind: string | null;
26
+ dtypeEffect: string | null;
27
+ reason: string | null;
28
+ evidence: string[];
29
+ } | null;
24
30
  }
25
31
 
26
32
  export interface ExecutionV1CompiledState {
@@ -9,6 +9,7 @@ import {
9
9
  } from './execution-runtime-builders.js';
10
10
  import { mergeKernelPathPolicy } from '../../../config/merge-helpers.js';
11
11
  import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
12
+ import { selectRuleValue } from '../../../rules/rule-registry.js';
12
13
  import { buildOpIdFromExecutionStep } from './operator-identity.js';
13
14
  import {
14
15
  resolveCapabilityTransforms,
@@ -32,6 +33,56 @@ function isSessionCapabilityTransform(name) {
32
33
  return SESSION_CAPABILITY_TRANSFORMS.has(name);
33
34
  }
34
35
 
36
+ function resolveExecutionCapabilities(capabilities) {
37
+ if (capabilities && typeof capabilities === 'object') {
38
+ return capabilities;
39
+ }
40
+ return {
41
+ hasSubgroups: false,
42
+ hasSubgroupsF16: false,
43
+ hasF16: false,
44
+ hasTimestampQuery: false,
45
+ maxBufferSize: 0,
46
+ maxWorkgroupSize: 0,
47
+ maxWorkgroupStorageSize: 0,
48
+ adapterInfo: {
49
+ vendor: 'unknown',
50
+ architecture: 'unknown',
51
+ device: 'unknown',
52
+ description: 'execution-v1-no-f16-proof',
53
+ },
54
+ };
55
+ }
56
+
57
+ function resolveExecutionSessionKVDtype(session, manifestInference, capabilities, useGPU) {
58
+ const runtimeKV = session?.kvcache ?? null;
59
+ if (!runtimeKV) {
60
+ return session;
61
+ }
62
+ if (useGPU == null && !capabilities) {
63
+ return session;
64
+ }
65
+ const attnSoftcap = manifestInference?.attention?.attnLogitSoftcapping;
66
+ const forceF32Softcap = runtimeKV.forceF32Softcap === true;
67
+ const forceF32KV = attnSoftcap != null && attnSoftcap > 0 && forceF32Softcap;
68
+ const kvDtype = selectRuleValue('inference', 'dtype', 'kvCacheDtype', {
69
+ requested: runtimeKV.kvDtype,
70
+ useGPU: useGPU === true,
71
+ hasF16: capabilities?.hasF16 === true,
72
+ forceF32: forceF32KV,
73
+ });
74
+ if (kvDtype === runtimeKV.kvDtype) {
75
+ return session;
76
+ }
77
+ return {
78
+ ...session,
79
+ kvcache: {
80
+ ...runtimeKV,
81
+ kvDtype,
82
+ },
83
+ };
84
+ }
85
+
35
86
  function mergeExecutionV1Session(manifestSession, runtimeSession) {
36
87
  return mergeRuntimeValues(
37
88
  manifestSession ?? {},
@@ -297,6 +348,34 @@ function preserveRuntimeDecodeLoop(updatedInference, runtimeConfig) {
297
348
  };
298
349
  }
299
350
 
351
+ function preserveConfiguredKernelPath(updatedInference, runtimeConfig) {
352
+ const configuredInference = runtimeConfig?.inference;
353
+ const configuredKernelPath = configuredInference?.kernelPath;
354
+ if (configuredKernelPath == null) {
355
+ return updatedInference;
356
+ }
357
+ const configuredSession = configuredInference?.session;
358
+ const hasConfiguredSessionCompute = hasOwnProperty(configuredSession, 'compute');
359
+ const hasConfiguredSessionKVCache = hasOwnProperty(configuredSession, 'kvcache');
360
+ return {
361
+ ...updatedInference,
362
+ kernelPath: configuredKernelPath,
363
+ kernelPathSource: 'config',
364
+ ...(hasOwnProperty(configuredInference, 'compute')
365
+ ? { compute: configuredInference.compute }
366
+ : {}),
367
+ ...(hasConfiguredSessionCompute || hasConfiguredSessionKVCache
368
+ ? {
369
+ session: {
370
+ ...updatedInference.session,
371
+ ...(hasConfiguredSessionCompute ? { compute: configuredSession.compute } : {}),
372
+ ...(hasConfiguredSessionKVCache ? { kvcache: configuredSession.kvcache } : {}),
373
+ },
374
+ }
375
+ : {}),
376
+ };
377
+ }
378
+
300
379
  const EXECUTION_V1_PROJECTION_OPS = new Set([
301
380
  'q_proj', 'k_proj', 'v_proj', 'o_proj',
302
381
  'gate_proj', 'up_proj', 'down_proj',
@@ -651,7 +730,16 @@ export function compileExecutionV1(options = {}) {
651
730
  const modelId = options.modelId ?? 'model';
652
731
  const numLayers = options.numLayers ?? 0;
653
732
  const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
654
- const capabilities = options.capabilities ?? null;
733
+ const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
734
+ const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
735
+ const useGPU = hasUseGPUOption
736
+ ? options.useGPU === true
737
+ : hasCapabilityProof
738
+ ? true
739
+ : null;
740
+ const capabilities = hasCapabilityProof || hasUseGPUOption
741
+ ? resolveExecutionCapabilities(options.capabilities ?? null)
742
+ : null;
655
743
  const platform = options.platform ?? null;
656
744
  const runtimeSession = options.runtimeSession ?? null;
657
745
  const runtimeCompute = options.runtimeCompute ?? null;
@@ -669,7 +757,13 @@ export function compileExecutionV1(options = {}) {
669
757
  manifestInference.session ?? {},
670
758
  runtimeSession ?? {}
671
759
  );
672
- const session = mergedSession;
760
+ const declaredSession = mergedSession;
761
+ const session = resolveExecutionSessionKVDtype(
762
+ mergedSession,
763
+ manifestInference,
764
+ capabilities,
765
+ useGPU
766
+ );
673
767
 
674
768
  if (!session?.compute?.defaults?.activationDtype) {
675
769
  throw new Error('[ExecutionV1] session.compute.defaults.activationDtype is required.');
@@ -678,7 +772,12 @@ export function compileExecutionV1(options = {}) {
678
772
  const activationDtype = session.compute.defaults.activationDtype;
679
773
  const mathDtype = session.compute.defaults.mathDtype ?? null;
680
774
  const accumDtype = session.compute.defaults.accumDtype ?? null;
775
+ const requestedActivationDtype = runtimeCompute?.activationDtype ?? activationDtype;
681
776
  const kvDtype = requireSessionKVDtype(session);
777
+ const declaredActivationDtype = declaredSession.compute?.defaults?.activationDtype ?? activationDtype;
778
+ const declaredMathDtype = declaredSession.compute?.defaults?.mathDtype ?? mathDtype;
779
+ const declaredAccumDtype = declaredSession.compute?.defaults?.accumDtype ?? accumDtype;
780
+ const declaredKvDtype = declaredSession.kvcache?.kvDtype ?? kvDtype;
682
781
  const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
683
782
 
684
783
  if (execution !== manifestExecution) {
@@ -690,7 +789,9 @@ export function compileExecutionV1(options = {}) {
690
789
  // -------------------------------------------------------------------------
691
790
  let appliedTransformNames = [];
692
791
  let graphWasTransformed = false;
792
+ let capabilityTransformPolicy = null;
693
793
  const graphContext = {
794
+ requestedActivationDtype,
694
795
  activationDtype,
695
796
  mathDtype,
696
797
  accumDtype,
@@ -704,6 +805,12 @@ export function compileExecutionV1(options = {}) {
704
805
 
705
806
  if (capabilities) {
706
807
  const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
808
+ capabilityTransformPolicy = {
809
+ kind: resolved.kind ?? null,
810
+ dtypeEffect: resolved.dtypeEffect ?? null,
811
+ reason: resolved.reason ?? null,
812
+ evidence: Array.isArray(resolved.evidence) ? [...resolved.evidence] : [],
813
+ };
707
814
  const sourceScope = kernelPathPolicy.sourceScope ?? kernelPathPolicy.allowSources ?? [];
708
815
  const remapAllowed = kernelPathPolicy.mode === 'capability-aware'
709
816
  && kernelPathPolicy.onIncompatible === 'remap'
@@ -845,10 +952,10 @@ export function compileExecutionV1(options = {}) {
845
952
  // any drift here is owned by capability transforms (e.g., widenToF32Activations
846
953
  // on hasF16=false GPUs).
847
954
  const declaredLane = {
848
- activationDtype,
849
- mathDtype,
850
- accumDtype,
851
- kvDtype,
955
+ activationDtype: declaredActivationDtype,
956
+ mathDtype: declaredMathDtype,
957
+ accumDtype: declaredAccumDtype,
958
+ kvDtype: declaredKvDtype,
852
959
  };
853
960
  const executedLane = {
854
961
  activationDtype: effectiveSession.compute?.defaults?.activationDtype ?? null,
@@ -867,6 +974,7 @@ export function compileExecutionV1(options = {}) {
867
974
  executed: executedLane,
868
975
  status: laneFieldDelta ? 'transformed' : 'matches',
869
976
  transforms: [...appliedTransformNames],
977
+ policy: capabilityTransformPolicy,
870
978
  };
871
979
 
872
980
  const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
@@ -914,6 +1022,9 @@ export function compileExecutionV1(options = {}) {
914
1022
  // until applyModelBatchingRuntimeDefaults in phase 2. If runtime batching was
915
1023
  // already explicitly configured, manifest decodeLoop is skipped and runtime
916
1024
  // values take precedence.
1025
+ // 3. preserveConfiguredKernelPath — restores a non-null runtime kernelPath and
1026
+ // its dtype-bearing config after compilation so explicit runtime config
1027
+ // remains the highest-precedence path contract.
917
1028
  //
918
1029
  // This function must be called exactly once per model load. Calling it again with
919
1030
  // an already-patched runtimeConfig would double-apply the execution-v1 merge and
@@ -954,8 +1065,11 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
954
1065
  });
955
1066
 
956
1067
  const runtimeInferencePatch = executionV1State.runtimeInferencePatch;
957
- const updatedInference = preserveRuntimeDecodeLoop(
958
- mergeRuntimeValues(runtimeConfig.inference ?? {}, runtimeInferencePatch),
1068
+ const updatedInference = preserveConfiguredKernelPath(
1069
+ preserveRuntimeDecodeLoop(
1070
+ mergeRuntimeValues(runtimeConfig.inference ?? {}, runtimeInferencePatch),
1071
+ runtimeConfig
1072
+ ),
959
1073
  runtimeConfig
960
1074
  );
961
1075
 
@@ -17,6 +17,8 @@ import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buf
17
17
  import {
18
18
  runFusedFFN,
19
19
  recordFusedFFN,
20
+ runFusedFFNFromRMSNormStats,
21
+ recordFusedFFNFromRMSNormStats,
20
22
  castF16ToF32,
21
23
  castF32ToF16,
22
24
  recordCastF16ToF32,
@@ -31,6 +33,7 @@ import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
31
33
  import { runProbes } from '../probes.js';
32
34
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
33
35
  import {
36
+ getKernelPathMatmulConstants,
34
37
  getKernelPathMatmulPrecision,
35
38
  getKernelPathMatmulVariant,
36
39
  } from '../../../../config/kernel-path-loader.js';
@@ -54,6 +57,93 @@ function isQ4KMatmulVariant(variant) {
54
57
  return typeof variant === 'string' && variant.startsWith('q4_');
55
58
  }
56
59
 
60
+ function normalizeFusedGateUpPipelineConstants(constants) {
61
+ if (!constants || typeof constants !== 'object') {
62
+ return null;
63
+ }
64
+ const colsPerWorkgroup = constants.COLS_PER_WG;
65
+ const threadsPerCol = constants.THREADS_PER_COL ?? constants.THREADS_PER_COL_GEMV;
66
+ const workgroupSize = constants.WORKGROUP_SIZE;
67
+ const useFullBlockFastPath = constants.USE_FULL_BLOCK_FAST_PATH;
68
+ if (
69
+ colsPerWorkgroup === undefined &&
70
+ threadsPerCol === undefined &&
71
+ workgroupSize === undefined &&
72
+ useFullBlockFastPath === undefined
73
+ ) {
74
+ return null;
75
+ }
76
+ return {
77
+ ...(workgroupSize !== undefined ? { WORKGROUP_SIZE: workgroupSize } : {}),
78
+ ...(colsPerWorkgroup !== undefined ? { COLS_PER_WG: colsPerWorkgroup } : {}),
79
+ ...(threadsPerCol !== undefined ? { THREADS_PER_COL: threadsPerCol } : {}),
80
+ ...(useFullBlockFastPath !== undefined ? { USE_FULL_BLOCK_FAST_PATH: useFullBlockFastPath } : {}),
81
+ };
82
+ }
83
+
84
+ function constantsEqual(a, b) {
85
+ const aKeys = Object.keys(a).sort();
86
+ const bKeys = Object.keys(b).sort();
87
+ if (aKeys.length !== bKeys.length) return false;
88
+ for (let i = 0; i < aKeys.length; i++) {
89
+ if (aKeys[i] !== bKeys[i] || a[aKeys[i]] !== b[bKeys[i]]) {
90
+ return false;
91
+ }
92
+ }
93
+ return true;
94
+ }
95
+
96
+ export function resolveFusedGateUpPipelineConstants(options = {}) {
97
+ const phase = options.phase ?? null;
98
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
99
+ const kernelPath = options.kernelPath ?? null;
100
+ if (!phase) {
101
+ return null;
102
+ }
103
+ if (kernelPath) {
104
+ const fusedConstants = normalizeFusedGateUpPipelineConstants(
105
+ getKernelPathMatmulConstants('ffn_gate_up', phase, layerIdx, kernelPath)
106
+ );
107
+ if (fusedConstants) {
108
+ return fusedConstants;
109
+ }
110
+ const gateConstants = normalizeFusedGateUpPipelineConstants(
111
+ getKernelPathMatmulConstants('ffn_gate', phase, layerIdx, kernelPath)
112
+ );
113
+ const upConstants = normalizeFusedGateUpPipelineConstants(
114
+ getKernelPathMatmulConstants('ffn_up', phase, layerIdx, kernelPath)
115
+ );
116
+ if (gateConstants || upConstants) {
117
+ if (!gateConstants || !upConstants || !constantsEqual(gateConstants, upConstants)) {
118
+ throw new Error(
119
+ `[FFN] Fused gate/up requires matching gate and up kernel constants; ` +
120
+ `got gate=${JSON.stringify(gateConstants)} up=${JSON.stringify(upConstants)}.`
121
+ );
122
+ }
123
+ return gateConstants;
124
+ }
125
+ }
126
+ const sessionConstants = getRuntimeConfig()
127
+ .inference?.session?.fusedFfnQ4K?.[phase]?.pipelineConstants;
128
+ return normalizeFusedGateUpPipelineConstants(sessionConstants);
129
+ }
130
+
131
+ export function resolveFusedGateUpVariant(options = {}) {
132
+ const phase = options.phase ?? null;
133
+ if (!phase) {
134
+ return null;
135
+ }
136
+ const variant = getRuntimeConfig()
137
+ .inference?.session?.fusedFfnQ4K?.[phase]?.variant;
138
+ if (variant == null) {
139
+ return null;
140
+ }
141
+ if (typeof variant !== 'string' || variant.length === 0) {
142
+ throw new Error(`[FFN] fusedFfnQ4K.${phase}.variant must be a non-empty string or null.`);
143
+ }
144
+ return variant;
145
+ }
146
+
57
147
  function enqueueRecordedDenseHealth(context, layerIdx, label, tensor, elementCount) {
58
148
  const recorder = context.recorder ?? null;
59
149
  if (!recorder || !isTraceEnabled('logits') || !shouldDebugLayerOutput(layerIdx, context.debugLayers)) {
@@ -265,6 +355,31 @@ export function resolveGateUpPathMode(options = {}) {
265
355
  return 'implicit';
266
356
  }
267
357
 
358
+ export function canFuseSplitPrefillF16GateUpPath(options = {}) {
359
+ const kernelPath = options.kernelPath ?? null;
360
+ const phase = options.phase ?? null;
361
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
362
+ if (!kernelPath || phase !== 'prefill') {
363
+ return false;
364
+ }
365
+ if (options.gateDtype !== 'f16' || options.upDtype !== 'f16') {
366
+ return false;
367
+ }
368
+ if (
369
+ hasExplicitMatmulPrecision('ffn_gate', phase, layerIdx, kernelPath)
370
+ || hasExplicitMatmulPrecision('ffn_up', phase, layerIdx, kernelPath)
371
+ || hasExplicitMatmulPrecision('ffn_down', phase, layerIdx, kernelPath)
372
+ ) {
373
+ return false;
374
+ }
375
+
376
+ const gateVariant = getKernelPathMatmulVariant('ffn_gate', phase, layerIdx, kernelPath);
377
+ const upVariant = getKernelPathMatmulVariant('ffn_up', phase, layerIdx, kernelPath);
378
+ return gateVariant != null
379
+ && gateVariant === upVariant
380
+ && !isQ4KMatmulVariant(gateVariant);
381
+ }
382
+
268
383
  export function resolveFusedGateUpWeights(layerWeights, options = {}) {
269
384
  const gate = layerWeights?.gate ?? null;
270
385
  const up = layerWeights?.up ?? null;
@@ -319,6 +434,9 @@ async function dispatchFusedGateUp({
319
434
  swigluLimit,
320
435
  recorder,
321
436
  executionPolicies = null,
437
+ normStats = null,
438
+ pipelineConstants = null,
439
+ variant = null,
322
440
  }) {
323
441
  const useNativeF16Fused = canUseNativeF16FusedGateUp({
324
442
  inputDtype: inputTensor.dtype,
@@ -344,16 +462,62 @@ async function dispatchFusedGateUp({
344
462
  }
345
463
 
346
464
  const activation = resolveActivationOp(hiddenActivation);
465
+ if (normStats) {
466
+ if (variant != null) {
467
+ throw new Error(`[FFN] explicit fused gate/up variant "${variant}" is not supported by the norm-stats path.`);
468
+ }
469
+ if (inputTensor.dtype !== 'f32' || gateDtype !== 'q4k' || numTokens !== 1) {
470
+ throw new Error(
471
+ `Fused post-attention norm gate/up requires f32 Q4_K decode input; ` +
472
+ `got input=${inputTensor.dtype}, gate=${gateDtype}, tokens=${numTokens}.`
473
+ );
474
+ }
475
+ const fusedNormedOutput = recorder
476
+ ? await recordFusedFFNFromRMSNormStats(
477
+ recorder,
478
+ inputTensor,
479
+ normStats.invRmsBuffer,
480
+ normStats.normWeight,
481
+ gateWeight,
482
+ upWeight,
483
+ hiddenSize,
484
+ intermediateSize,
485
+ {
486
+ batchSize: numTokens,
487
+ activation,
488
+ swigluLimit,
489
+ rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
490
+ pipelineConstants,
491
+ }
492
+ )
493
+ : await runFusedFFNFromRMSNormStats(
494
+ inputTensor,
495
+ normStats.invRmsBuffer,
496
+ normStats.normWeight,
497
+ gateWeight,
498
+ upWeight,
499
+ hiddenSize,
500
+ intermediateSize,
501
+ {
502
+ batchSize: numTokens,
503
+ activation,
504
+ swigluLimit,
505
+ rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
506
+ pipelineConstants,
507
+ }
508
+ );
509
+ return fusedNormedOutput;
510
+ }
347
511
  const fusedOutput = recorder
348
512
  ? await recordFusedFFN(
349
513
  recorder, fusedInput, gateWeight, upWeight,
350
514
  hiddenSize, intermediateSize,
351
- { batchSize: numTokens, activation, swigluLimit }
515
+ { batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
352
516
  )
353
517
  : await runFusedFFN(
354
518
  fusedInput, gateWeight, upWeight,
355
519
  hiddenSize, intermediateSize,
356
- { batchSize: numTokens, activation, swigluLimit }
520
+ { batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
357
521
  );
358
522
 
359
523
  if (!recorder && fusedInput !== inputTensor) {
@@ -621,18 +785,29 @@ export async function runDenseFFNGPU(
621
785
  hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
622
786
  activationDtype,
623
787
  f16BatchSupported,
788
+ useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
624
789
  batchSize: numTokens,
625
790
  hiddenSizeAligned32,
626
791
  useDoubleWideMlp: Boolean(useDoubleWideMlp),
627
792
  });
628
- const useFusedGateUp = gateUpPathMode === 'split'
793
+ const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
794
+ && canFuseSplitPrefillF16GateUpPath({
795
+ kernelPath,
796
+ phase,
797
+ layerIdx,
798
+ gateDtype,
799
+ upDtype,
800
+ });
801
+ const useFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
629
802
  ? false
630
803
  : useFusedGateUpByRule;
631
804
  trace.ffn(
632
805
  layerIdx,
633
- `useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
806
+ `useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
807
+ `inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
634
808
  `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
635
- `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
809
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
810
+ `largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
636
811
  );
637
812
 
638
813
  if (useFusedGateUp) {
@@ -650,6 +825,12 @@ export async function runDenseFFNGPU(
650
825
  const gateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
651
826
  const upWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
652
827
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
828
+ const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
829
+ kernelPath,
830
+ phase,
831
+ layerIdx,
832
+ });
833
+ const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
653
834
  const fusedDownOutputDtype = resolveMatmulStepDtype(
654
835
  'ffn_down',
655
836
  phase,
@@ -674,6 +855,18 @@ export async function runDenseFFNGPU(
674
855
  hiddenSize, intermediateSize, numTokens,
675
856
  hiddenActivation, swigluLimit, recorder,
676
857
  executionPolicies: context.executionPolicies ?? null,
858
+ normStats: context.__pendingFfnInputNormStats ?? null,
859
+ pipelineConstants: fusedGateUpPipelineConstants,
860
+ variant: fusedGateUpVariant,
861
+ });
862
+ await runProbes('ffn_act', fusedOutput.buffer, {
863
+ layerIdx,
864
+ numTokens,
865
+ hiddenSize: intermediateSize,
866
+ probes: context.debugProbes,
867
+ recorder,
868
+ operatorDiagnostics: context.operatorDiagnostics,
869
+ dtype: fusedOutput.dtype,
677
870
  });
678
871
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
679
872
 
@@ -778,6 +971,12 @@ export async function runDenseFFNGPU(
778
971
  return output;
779
972
  }
780
973
 
974
+ if (context.__pendingFfnInputNormStats) {
975
+ throw new Error(
976
+ `Layer ${layerIdx} has precomputed FFN input norm stats, but fused gate/up was not selected.`
977
+ );
978
+ }
979
+
781
980
  if (!layerWeights?.gate || !layerWeights?.up || !layerWeights?.down) {
782
981
  log.warn('Layer', `L${layerIdx} FFN: no weights found`);
783
982
  const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: inputTensor.dtype });
@@ -1340,30 +1539,47 @@ export async function runDenseFFNWithFusedPostNormGPU(
1340
1539
  hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
1341
1540
  activationDtype,
1342
1541
  f16BatchSupported: getKernelCapabilities().hasF16,
1542
+ useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
1343
1543
  batchSize: numTokens,
1344
1544
  hiddenSizeAligned32,
1345
1545
  useDoubleWideMlp: Boolean(useDoubleWideMlp),
1346
1546
  });
1347
- const canUseFusedGateUp = gateUpPathMode === 'split'
1547
+ const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
1548
+ && canFuseSplitPrefillF16GateUpPath({
1549
+ kernelPath,
1550
+ phase,
1551
+ layerIdx,
1552
+ gateDtype,
1553
+ upDtype,
1554
+ });
1555
+ const canUseFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
1348
1556
  ? false
1349
1557
  : canUseFusedGateUpByRule;
1350
1558
  trace.ffn(
1351
1559
  layerIdx,
1352
- `useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
1560
+ `useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
1561
+ `inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
1353
1562
  `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
1354
- `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
1563
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
1564
+ `largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
1355
1565
  );
1356
1566
  const gateWeight = canUseFusedGateUp
1357
1567
  ? fusedGateWeight
1358
1568
  : getWeightBuffer(layerWeights.gate, 'ffn_gate');
1359
- const upWeight = canUseFusedGateUp
1360
- ? fusedUpWeight
1361
- : getWeightBuffer(layerWeights.up, 'ffn_up');
1362
-
1363
- if (canUseFusedGateUp) {
1364
- const {
1365
- fusedGateUpInputDtype,
1366
- } = resolveDenseFFNFusedPathDtypes({
1569
+ const upWeight = canUseFusedGateUp
1570
+ ? fusedUpWeight
1571
+ : getWeightBuffer(layerWeights.up, 'ffn_up');
1572
+
1573
+ if (canUseFusedGateUp) {
1574
+ const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
1575
+ kernelPath,
1576
+ phase,
1577
+ layerIdx,
1578
+ });
1579
+ const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
1580
+ const {
1581
+ fusedGateUpInputDtype,
1582
+ } = resolveDenseFFNFusedPathDtypes({
1367
1583
  phase,
1368
1584
  layerIdx,
1369
1585
  kernelPath,
@@ -1382,11 +1598,13 @@ export async function runDenseFFNWithFusedPostNormGPU(
1382
1598
  fusedInputOwned = fusedInput !== inputTensor;
1383
1599
  }
1384
1600
  activatedOutput = await dispatchFusedGateUp({
1385
- inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
1386
- hiddenSize, intermediateSize, numTokens,
1387
- hiddenActivation, swigluLimit, recorder,
1388
- executionPolicies: context.executionPolicies ?? null,
1389
- });
1601
+ inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
1602
+ hiddenSize, intermediateSize, numTokens,
1603
+ hiddenActivation, swigluLimit, recorder,
1604
+ executionPolicies: context.executionPolicies ?? null,
1605
+ pipelineConstants: fusedGateUpPipelineConstants,
1606
+ variant: fusedGateUpVariant,
1607
+ });
1390
1608
  if (fusedInputOwned) {
1391
1609
  if (recorder) {
1392
1610
  recorder.trackTemporaryBuffer(fusedInput.buffer);