doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -17,6 +17,8 @@ import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buf
17
17
  import {
18
18
  runFusedFFN,
19
19
  recordFusedFFN,
20
+ runFusedFFNFromRMSNormStats,
21
+ recordFusedFFNFromRMSNormStats,
20
22
  castF16ToF32,
21
23
  castF32ToF16,
22
24
  recordCastF16ToF32,
@@ -31,6 +33,7 @@ import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
31
33
  import { runProbes } from '../probes.js';
32
34
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
33
35
  import {
36
+ getKernelPathMatmulConstants,
34
37
  getKernelPathMatmulPrecision,
35
38
  getKernelPathMatmulVariant,
36
39
  } from '../../../../config/kernel-path-loader.js';
@@ -54,6 +57,93 @@ function isQ4KMatmulVariant(variant) {
54
57
  return typeof variant === 'string' && variant.startsWith('q4_');
55
58
  }
56
59
 
60
+ function normalizeFusedGateUpPipelineConstants(constants) {
61
+ if (!constants || typeof constants !== 'object') {
62
+ return null;
63
+ }
64
+ const colsPerWorkgroup = constants.COLS_PER_WG;
65
+ const threadsPerCol = constants.THREADS_PER_COL ?? constants.THREADS_PER_COL_GEMV;
66
+ const workgroupSize = constants.WORKGROUP_SIZE;
67
+ const useFullBlockFastPath = constants.USE_FULL_BLOCK_FAST_PATH;
68
+ if (
69
+ colsPerWorkgroup === undefined &&
70
+ threadsPerCol === undefined &&
71
+ workgroupSize === undefined &&
72
+ useFullBlockFastPath === undefined
73
+ ) {
74
+ return null;
75
+ }
76
+ return {
77
+ ...(workgroupSize !== undefined ? { WORKGROUP_SIZE: workgroupSize } : {}),
78
+ ...(colsPerWorkgroup !== undefined ? { COLS_PER_WG: colsPerWorkgroup } : {}),
79
+ ...(threadsPerCol !== undefined ? { THREADS_PER_COL: threadsPerCol } : {}),
80
+ ...(useFullBlockFastPath !== undefined ? { USE_FULL_BLOCK_FAST_PATH: useFullBlockFastPath } : {}),
81
+ };
82
+ }
83
+
84
+ function constantsEqual(a, b) {
85
+ const aKeys = Object.keys(a).sort();
86
+ const bKeys = Object.keys(b).sort();
87
+ if (aKeys.length !== bKeys.length) return false;
88
+ for (let i = 0; i < aKeys.length; i++) {
89
+ if (aKeys[i] !== bKeys[i] || a[aKeys[i]] !== b[bKeys[i]]) {
90
+ return false;
91
+ }
92
+ }
93
+ return true;
94
+ }
95
+
96
+ export function resolveFusedGateUpPipelineConstants(options = {}) {
97
+ const phase = options.phase ?? null;
98
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
99
+ const kernelPath = options.kernelPath ?? null;
100
+ if (!phase) {
101
+ return null;
102
+ }
103
+ if (kernelPath) {
104
+ const fusedConstants = normalizeFusedGateUpPipelineConstants(
105
+ getKernelPathMatmulConstants('ffn_gate_up', phase, layerIdx, kernelPath)
106
+ );
107
+ if (fusedConstants) {
108
+ return fusedConstants;
109
+ }
110
+ const gateConstants = normalizeFusedGateUpPipelineConstants(
111
+ getKernelPathMatmulConstants('ffn_gate', phase, layerIdx, kernelPath)
112
+ );
113
+ const upConstants = normalizeFusedGateUpPipelineConstants(
114
+ getKernelPathMatmulConstants('ffn_up', phase, layerIdx, kernelPath)
115
+ );
116
+ if (gateConstants || upConstants) {
117
+ if (!gateConstants || !upConstants || !constantsEqual(gateConstants, upConstants)) {
118
+ throw new Error(
119
+ `[FFN] Fused gate/up requires matching gate and up kernel constants; ` +
120
+ `got gate=${JSON.stringify(gateConstants)} up=${JSON.stringify(upConstants)}.`
121
+ );
122
+ }
123
+ return gateConstants;
124
+ }
125
+ }
126
+ const sessionConstants = getRuntimeConfig()
127
+ .inference?.session?.fusedFfnQ4K?.[phase]?.pipelineConstants;
128
+ return normalizeFusedGateUpPipelineConstants(sessionConstants);
129
+ }
130
+
131
+ export function resolveFusedGateUpVariant(options = {}) {
132
+ const phase = options.phase ?? null;
133
+ if (!phase) {
134
+ return null;
135
+ }
136
+ const variant = getRuntimeConfig()
137
+ .inference?.session?.fusedFfnQ4K?.[phase]?.variant;
138
+ if (variant == null) {
139
+ return null;
140
+ }
141
+ if (typeof variant !== 'string' || variant.length === 0) {
142
+ throw new Error(`[FFN] fusedFfnQ4K.${phase}.variant must be a non-empty string or null.`);
143
+ }
144
+ return variant;
145
+ }
146
+
57
147
  function enqueueRecordedDenseHealth(context, layerIdx, label, tensor, elementCount) {
58
148
  const recorder = context.recorder ?? null;
59
149
  if (!recorder || !isTraceEnabled('logits') || !shouldDebugLayerOutput(layerIdx, context.debugLayers)) {
@@ -265,6 +355,31 @@ export function resolveGateUpPathMode(options = {}) {
265
355
  return 'implicit';
266
356
  }
267
357
 
358
+ export function canFuseSplitPrefillF16GateUpPath(options = {}) {
359
+ const kernelPath = options.kernelPath ?? null;
360
+ const phase = options.phase ?? null;
361
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
362
+ if (!kernelPath || phase !== 'prefill') {
363
+ return false;
364
+ }
365
+ if (options.gateDtype !== 'f16' || options.upDtype !== 'f16') {
366
+ return false;
367
+ }
368
+ if (
369
+ hasExplicitMatmulPrecision('ffn_gate', phase, layerIdx, kernelPath)
370
+ || hasExplicitMatmulPrecision('ffn_up', phase, layerIdx, kernelPath)
371
+ || hasExplicitMatmulPrecision('ffn_down', phase, layerIdx, kernelPath)
372
+ ) {
373
+ return false;
374
+ }
375
+
376
+ const gateVariant = getKernelPathMatmulVariant('ffn_gate', phase, layerIdx, kernelPath);
377
+ const upVariant = getKernelPathMatmulVariant('ffn_up', phase, layerIdx, kernelPath);
378
+ return gateVariant != null
379
+ && gateVariant === upVariant
380
+ && !isQ4KMatmulVariant(gateVariant);
381
+ }
382
+
268
383
  export function resolveFusedGateUpWeights(layerWeights, options = {}) {
269
384
  const gate = layerWeights?.gate ?? null;
270
385
  const up = layerWeights?.up ?? null;
@@ -319,6 +434,9 @@ async function dispatchFusedGateUp({
319
434
  swigluLimit,
320
435
  recorder,
321
436
  executionPolicies = null,
437
+ normStats = null,
438
+ pipelineConstants = null,
439
+ variant = null,
322
440
  }) {
323
441
  const useNativeF16Fused = canUseNativeF16FusedGateUp({
324
442
  inputDtype: inputTensor.dtype,
@@ -344,16 +462,62 @@ async function dispatchFusedGateUp({
344
462
  }
345
463
 
346
464
  const activation = resolveActivationOp(hiddenActivation);
465
+ if (normStats) {
466
+ if (variant != null) {
467
+ throw new Error(`[FFN] explicit fused gate/up variant "${variant}" is not supported by the norm-stats path.`);
468
+ }
469
+ if (inputTensor.dtype !== 'f32' || gateDtype !== 'q4k' || numTokens !== 1) {
470
+ throw new Error(
471
+ `Fused post-attention norm gate/up requires f32 Q4_K decode input; ` +
472
+ `got input=${inputTensor.dtype}, gate=${gateDtype}, tokens=${numTokens}.`
473
+ );
474
+ }
475
+ const fusedNormedOutput = recorder
476
+ ? await recordFusedFFNFromRMSNormStats(
477
+ recorder,
478
+ inputTensor,
479
+ normStats.invRmsBuffer,
480
+ normStats.normWeight,
481
+ gateWeight,
482
+ upWeight,
483
+ hiddenSize,
484
+ intermediateSize,
485
+ {
486
+ batchSize: numTokens,
487
+ activation,
488
+ swigluLimit,
489
+ rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
490
+ pipelineConstants,
491
+ }
492
+ )
493
+ : await runFusedFFNFromRMSNormStats(
494
+ inputTensor,
495
+ normStats.invRmsBuffer,
496
+ normStats.normWeight,
497
+ gateWeight,
498
+ upWeight,
499
+ hiddenSize,
500
+ intermediateSize,
501
+ {
502
+ batchSize: numTokens,
503
+ activation,
504
+ swigluLimit,
505
+ rmsNormWeightOffset: normStats.rmsNormWeightOffset === true,
506
+ pipelineConstants,
507
+ }
508
+ );
509
+ return fusedNormedOutput;
510
+ }
347
511
  const fusedOutput = recorder
348
512
  ? await recordFusedFFN(
349
513
  recorder, fusedInput, gateWeight, upWeight,
350
514
  hiddenSize, intermediateSize,
351
- { batchSize: numTokens, activation, swigluLimit }
515
+ { batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
352
516
  )
353
517
  : await runFusedFFN(
354
518
  fusedInput, gateWeight, upWeight,
355
519
  hiddenSize, intermediateSize,
356
- { batchSize: numTokens, activation, swigluLimit }
520
+ { batchSize: numTokens, activation, swigluLimit, pipelineConstants, variant }
357
521
  );
358
522
 
359
523
  if (!recorder && fusedInput !== inputTensor) {
@@ -621,18 +785,29 @@ export async function runDenseFFNGPU(
621
785
  hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
622
786
  activationDtype,
623
787
  f16BatchSupported,
788
+ useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
624
789
  batchSize: numTokens,
625
790
  hiddenSizeAligned32,
626
791
  useDoubleWideMlp: Boolean(useDoubleWideMlp),
627
792
  });
628
- const useFusedGateUp = gateUpPathMode === 'split'
793
+ const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
794
+ && canFuseSplitPrefillF16GateUpPath({
795
+ kernelPath,
796
+ phase,
797
+ layerIdx,
798
+ gateDtype,
799
+ upDtype,
800
+ });
801
+ const useFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
629
802
  ? false
630
803
  : useFusedGateUpByRule;
631
804
  trace.ffn(
632
805
  layerIdx,
633
- `useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
806
+ `useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
807
+ `inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
634
808
  `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
635
- `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
809
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
810
+ `largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
636
811
  );
637
812
 
638
813
  if (useFusedGateUp) {
@@ -650,6 +825,12 @@ export async function runDenseFFNGPU(
650
825
  const gateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
651
826
  const upWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
652
827
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
828
+ const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
829
+ kernelPath,
830
+ phase,
831
+ layerIdx,
832
+ });
833
+ const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
653
834
  const fusedDownOutputDtype = resolveMatmulStepDtype(
654
835
  'ffn_down',
655
836
  phase,
@@ -674,6 +855,18 @@ export async function runDenseFFNGPU(
674
855
  hiddenSize, intermediateSize, numTokens,
675
856
  hiddenActivation, swigluLimit, recorder,
676
857
  executionPolicies: context.executionPolicies ?? null,
858
+ normStats: context.__pendingFfnInputNormStats ?? null,
859
+ pipelineConstants: fusedGateUpPipelineConstants,
860
+ variant: fusedGateUpVariant,
861
+ });
862
+ await runProbes('ffn_act', fusedOutput.buffer, {
863
+ layerIdx,
864
+ numTokens,
865
+ hiddenSize: intermediateSize,
866
+ probes: context.debugProbes,
867
+ recorder,
868
+ operatorDiagnostics: context.operatorDiagnostics,
869
+ dtype: fusedOutput.dtype,
677
870
  });
678
871
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_fused_gate_up', fusedOutput, numTokens * intermediateSize);
679
872
 
@@ -778,6 +971,12 @@ export async function runDenseFFNGPU(
778
971
  return output;
779
972
  }
780
973
 
974
+ if (context.__pendingFfnInputNormStats) {
975
+ throw new Error(
976
+ `Layer ${layerIdx} has precomputed FFN input norm stats, but fused gate/up was not selected.`
977
+ );
978
+ }
979
+
781
980
  if (!layerWeights?.gate || !layerWeights?.up || !layerWeights?.down) {
782
981
  log.warn('Layer', `L${layerIdx} FFN: no weights found`);
783
982
  const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: inputTensor.dtype });
@@ -1058,6 +1257,7 @@ export async function runDenseFFNGPU(
1058
1257
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1059
1258
  size: numTokens * intermediateSize,
1060
1259
  gate: gateOutput,
1260
+ inputActivation: 'identity',
1061
1261
  swigluLimit,
1062
1262
  label: `L${layerIdx}.ffn_activation`,
1063
1263
  layerIdx,
@@ -1340,30 +1540,47 @@ export async function runDenseFFNWithFusedPostNormGPU(
1340
1540
  hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
1341
1541
  activationDtype,
1342
1542
  f16BatchSupported: getKernelCapabilities().hasF16,
1543
+ useLargeBatchF16F32FusedGateUp: context.useLargeBatchF16F32FusedGateUp === true,
1343
1544
  batchSize: numTokens,
1344
1545
  hiddenSizeAligned32,
1345
1546
  useDoubleWideMlp: Boolean(useDoubleWideMlp),
1346
1547
  });
1347
- const canUseFusedGateUp = gateUpPathMode === 'split'
1548
+ const splitPrefillF16FusionAllowed = gateUpPathMode === 'split'
1549
+ && canFuseSplitPrefillF16GateUpPath({
1550
+ kernelPath,
1551
+ phase,
1552
+ layerIdx,
1553
+ gateDtype,
1554
+ upDtype,
1555
+ });
1556
+ const canUseFusedGateUp = gateUpPathMode === 'split' && !splitPrefillF16FusionAllowed
1348
1557
  ? false
1349
1558
  : canUseFusedGateUpByRule;
1350
1559
  trace.ffn(
1351
1560
  layerIdx,
1352
- `useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
1561
+ `useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} splitPrefillF16FusionAllowed=${splitPrefillF16FusionAllowed} ` +
1562
+ `inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
1353
1563
  `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
1354
- `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
1564
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} ` +
1565
+ `largeBatchF16F32FusedGateUp=${context.useLargeBatchF16F32FusedGateUp === true} batchSize=${numTokens}`
1355
1566
  );
1356
1567
  const gateWeight = canUseFusedGateUp
1357
1568
  ? fusedGateWeight
1358
1569
  : getWeightBuffer(layerWeights.gate, 'ffn_gate');
1359
- const upWeight = canUseFusedGateUp
1360
- ? fusedUpWeight
1361
- : getWeightBuffer(layerWeights.up, 'ffn_up');
1362
-
1363
- if (canUseFusedGateUp) {
1364
- const {
1365
- fusedGateUpInputDtype,
1366
- } = resolveDenseFFNFusedPathDtypes({
1570
+ const upWeight = canUseFusedGateUp
1571
+ ? fusedUpWeight
1572
+ : getWeightBuffer(layerWeights.up, 'ffn_up');
1573
+
1574
+ if (canUseFusedGateUp) {
1575
+ const fusedGateUpPipelineConstants = resolveFusedGateUpPipelineConstants({
1576
+ kernelPath,
1577
+ phase,
1578
+ layerIdx,
1579
+ });
1580
+ const fusedGateUpVariant = resolveFusedGateUpVariant({ phase });
1581
+ const {
1582
+ fusedGateUpInputDtype,
1583
+ } = resolveDenseFFNFusedPathDtypes({
1367
1584
  phase,
1368
1585
  layerIdx,
1369
1586
  kernelPath,
@@ -1382,11 +1599,13 @@ export async function runDenseFFNWithFusedPostNormGPU(
1382
1599
  fusedInputOwned = fusedInput !== inputTensor;
1383
1600
  }
1384
1601
  activatedOutput = await dispatchFusedGateUp({
1385
- inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
1386
- hiddenSize, intermediateSize, numTokens,
1387
- hiddenActivation, swigluLimit, recorder,
1388
- executionPolicies: context.executionPolicies ?? null,
1389
- });
1602
+ inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
1603
+ hiddenSize, intermediateSize, numTokens,
1604
+ hiddenActivation, swigluLimit, recorder,
1605
+ executionPolicies: context.executionPolicies ?? null,
1606
+ pipelineConstants: fusedGateUpPipelineConstants,
1607
+ variant: fusedGateUpVariant,
1608
+ });
1390
1609
  if (fusedInputOwned) {
1391
1610
  if (recorder) {
1392
1611
  recorder.trackTemporaryBuffer(fusedInput.buffer);
@@ -1428,6 +1647,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
1428
1647
  activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
1429
1648
  size: numTokens * intermediateSize,
1430
1649
  gate: gateOutput,
1650
+ inputActivation: 'identity',
1431
1651
  swigluLimit,
1432
1652
  }, recorder);
1433
1653
 
@@ -1,6 +1,6 @@
1
1
 
2
2
 
3
- import { doCast, doRMSNorm, doResidualAdd, releaseOrTrack } from '../ops.js';
3
+ import { doCast, doRMSNorm, doRMSNormStats, doResidualAdd, doResidualNextRMSNormPair, releaseOrTrack } from '../ops.js';
4
4
  import { getNormWeightBuffer } from '../weights.js';
5
5
  import { runProbes } from '../probes.js';
6
6
  import { isMoELayerLocal } from './types.js';
@@ -12,6 +12,17 @@ import { isGpuBufferInstance, isWeightBuffer } from '../../../../gpu/weight-buff
12
12
  import { shouldDebugLayerOutput, decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
13
13
  import { trace, isTraceEnabled } from '../../../../debug/index.js';
14
14
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
15
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
16
+
17
+ function shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context }) {
18
+ if (session?.usePostAttnNormFusedGateUp !== true) return false;
19
+ if (numTokens !== 1) return false;
20
+ if (postAttn?.dtype !== 'f32' || fusedResidualInput?.dtype !== 'f32') return false;
21
+ if (context.debugProbes?.length) return false;
22
+ if (context.debugCheckBuffer) return false;
23
+ if (isTraceEnabled('logits')) return false;
24
+ return true;
25
+ }
15
26
 
16
27
  async function debugFFNBuffer(context, layerIdx, label, tensor, numTokens, hiddenSize) {
17
28
  if (!context.debugCheckBuffer) return;
@@ -46,7 +57,8 @@ export async function processFFNStandard(
46
57
  layerWeights,
47
58
  fusedResidualInput,
48
59
  finalOutputScale = 1,
49
- residualBranchScale = 1
60
+ residualBranchScale = 1,
61
+ nextInputNormFusion = null
50
62
  ) {
51
63
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
52
64
  const { hiddenSize, rmsNormEps } = config;
@@ -70,22 +82,39 @@ export async function processFFNStandard(
70
82
  // 1. Post-attention norm (optionally fuses upstream residual add via PRE_RESIDUAL)
71
83
  let normedTensor = postAttn;
72
84
  let prenormSumBuffer = null;
85
+ let postAttnNormStats = null;
86
+ let postAttnNormWeightBuf = null;
87
+ let releasePostAttnNormWeight = false;
88
+ let usedPostAttnNormStatsPrelude = false;
73
89
  if (layerWeights?.postAttnNorm) {
74
90
  const normWeightBuf = getNormWeightBuffer(layerWeights.postAttnNorm, 'post_attn_norm', weightConfig, debugFlags);
91
+ postAttnNormWeightBuf = normWeightBuf;
92
+ releasePostAttnNormWeight = !isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm);
93
+ const session = getRuntimeConfig().inference?.session;
75
94
 
76
95
  if (fusedResidualInput) {
77
- // Fused path: rmsnorm(postAttn + fusedResidualInput) and write pre-norm sum
78
96
  const bytesPerElement = postAttn.dtype === 'f16' ? 2 : 4;
79
97
  prenormSumBuffer = acquireBuffer(size * bytesPerElement, undefined, 'fused_prenorm_sum');
80
- normedTensor = await doRMSNorm(postAttn, normWeightBuf, rmsNormEps, {
81
- batchSize: numTokens,
82
- hiddenSize,
83
- preResidual: fusedResidualInput,
84
- residualSumOutput: prenormSumBuffer,
85
- label: `L${layerIdx}.post_attn_norm`,
86
- layerIdx,
87
- rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
88
- }, recorder);
98
+ if (shouldUsePostAttnNormFusedGateUpPrelude({ session, postAttn, fusedResidualInput, numTokens, context })) {
99
+ postAttnNormStats = await doRMSNormStats(postAttn, fusedResidualInput, rmsNormEps, {
100
+ batchSize: numTokens,
101
+ hiddenSize,
102
+ outputBuffer: prenormSumBuffer,
103
+ label: `L${layerIdx}.post_attn_norm_stats`,
104
+ }, recorder);
105
+ normedTensor = postAttnNormStats.prenormSum;
106
+ usedPostAttnNormStatsPrelude = true;
107
+ } else {
108
+ normedTensor = await doRMSNorm(postAttn, normWeightBuf, rmsNormEps, {
109
+ batchSize: numTokens,
110
+ hiddenSize,
111
+ preResidual: fusedResidualInput,
112
+ residualSumOutput: prenormSumBuffer,
113
+ label: `L${layerIdx}.post_attn_norm`,
114
+ layerIdx,
115
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
116
+ }, recorder);
117
+ }
89
118
  } else {
90
119
  normedTensor = await doRMSNorm(postAttn, normWeightBuf, rmsNormEps, {
91
120
  batchSize: numTokens,
@@ -95,20 +124,20 @@ export async function processFFNStandard(
95
124
  rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
96
125
  }, recorder);
97
126
  }
98
-
99
- if (!isGpuBufferInstance(layerWeights.postAttnNorm) && !isWeightBuffer(layerWeights.postAttnNorm)) releaseOrTrack(recorder, normWeightBuf);
100
127
  }
101
- await runProbes('ffn_in', normedTensor.buffer, {
102
- layerIdx,
103
- numTokens,
104
- hiddenSize,
105
- probes: context.debugProbes,
106
- recorder,
107
- operatorDiagnostics: context.operatorDiagnostics,
108
- dtype: normedTensor.dtype,
109
- });
110
- await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
111
- enqueueRecordedFFNHealth(context, layerIdx, 'ffn_in', normedTensor, numTokens * hiddenSize);
128
+ if (!usedPostAttnNormStatsPrelude) {
129
+ await runProbes('ffn_in', normedTensor.buffer, {
130
+ layerIdx,
131
+ numTokens,
132
+ hiddenSize,
133
+ probes: context.debugProbes,
134
+ recorder,
135
+ operatorDiagnostics: context.operatorDiagnostics,
136
+ dtype: normedTensor.dtype,
137
+ });
138
+ await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
139
+ enqueueRecordedFFNHealth(context, layerIdx, 'ffn_in', normedTensor, numTokens * hiddenSize);
140
+ }
112
141
 
113
142
  // 2. FFN
114
143
  // Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
@@ -124,6 +153,13 @@ export async function processFFNStandard(
124
153
  context.__pendingFfnResidualTensor = requestedResidualBranchScale === 1
125
154
  ? pendingResidualForFfn
126
155
  : null;
156
+ context.__pendingFfnInputNormStats = postAttnNormStats
157
+ ? {
158
+ invRmsBuffer: postAttnNormStats.invRmsBuffer,
159
+ normWeight: postAttnNormWeightBuf,
160
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
161
+ }
162
+ : null;
127
163
  context.__ffnResidualFusedFired = false;
128
164
 
129
165
  let ffnOutput;
@@ -131,13 +167,17 @@ export async function processFFNStandard(
131
167
  && config.ffnBranchMode !== 'dense_plus_moe'
132
168
  && config.useMoE
133
169
  && isMoELayerLocal(layerIdx, config, layerWeights);
134
- if (useLegacyMoeOnlyFFN) {
135
- ffnOutput = await runMoEFFNGPU(layerIdx, normedTensor, numTokens, context);
136
- } else {
137
- ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
170
+ try {
171
+ if (useLegacyMoeOnlyFFN) {
172
+ ffnOutput = await runMoEFFNGPU(layerIdx, normedTensor, numTokens, context);
173
+ } else {
174
+ ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
175
+ }
176
+ } finally {
177
+ context.__pendingFfnResidualTensor = null;
178
+ context.__pendingFfnInputNormStats = null;
138
179
  }
139
180
  const ffnResidualFused = context.__ffnResidualFusedFired === true;
140
- context.__pendingFfnResidualTensor = null;
141
181
  context.__ffnResidualFusedFired = false;
142
182
  await runProbes('ffn_out', ffnOutput.buffer, {
143
183
  layerIdx,
@@ -184,13 +224,50 @@ export async function processFFNStandard(
184
224
  && !context.debugProbes?.length
185
225
  ? requestedFinalOutputScale
186
226
  : 1;
187
- output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
188
- label: `L${layerIdx}.ffn_residual`,
189
- layerIdx,
190
- outputBuffer: decodeOutputBuffer,
191
- outputScale: residualOutputScale,
192
- executionPolicies: context.executionPolicies ?? null,
193
- });
227
+ if (
228
+ nextInputNormFusion
229
+ && residualInput.dtype === 'f32'
230
+ && residualTensor.dtype === 'f32'
231
+ ) {
232
+ const normWeightBuf = getNormWeightBuffer(
233
+ nextInputNormFusion.weight,
234
+ 'next_input_norm',
235
+ weightConfig,
236
+ debugFlags
237
+ );
238
+ const pair = await doResidualNextRMSNormPair(
239
+ residualInput,
240
+ residualTensor,
241
+ normWeightBuf,
242
+ rmsNormEps,
243
+ {
244
+ batchSize: numTokens,
245
+ hiddenSize,
246
+ residualOutputBuffer: decodeOutputBuffer,
247
+ outputScale: residualOutputScale,
248
+ label: `L${layerIdx}.ffn_residual_next_input_norm`,
249
+ layerIdx,
250
+ rmsNormWeightOffset: weightConfig.rmsNormWeightOffset,
251
+ },
252
+ recorder
253
+ );
254
+ output = pair.residual;
255
+ context.__precomputedInputNorm = {
256
+ layerIdx: nextInputNormFusion.layerIdx,
257
+ tensor: pair.nextInput,
258
+ };
259
+ if (!isGpuBufferInstance(nextInputNormFusion.weight) && !isWeightBuffer(nextInputNormFusion.weight)) {
260
+ releaseOrTrack(recorder, normWeightBuf);
261
+ }
262
+ } else {
263
+ output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
264
+ label: `L${layerIdx}.ffn_residual`,
265
+ layerIdx,
266
+ outputBuffer: decodeOutputBuffer,
267
+ outputScale: residualOutputScale,
268
+ executionPolicies: context.executionPolicies ?? null,
269
+ });
270
+ }
194
271
  if (residualOutputScale !== 1) {
195
272
  context.__layerScalarFusedFired = true;
196
273
  }
@@ -207,9 +284,15 @@ export async function processFFNStandard(
207
284
  await debugFFNBuffer(context, layerIdx, 'layer output', output, numTokens, hiddenSize);
208
285
  enqueueRecordedFFNHealth(context, layerIdx, 'layer_out', output, numTokens * hiddenSize);
209
286
 
210
- if (normedTensor !== postAttn) {
287
+ if (normedTensor !== postAttn && !usedPostAttnNormStatsPrelude) {
211
288
  releaseOrTrack(recorder, normedTensor.buffer, decodeBuffers);
212
289
  }
290
+ if (postAttnNormStats?.invRmsBuffer) {
291
+ releaseOrTrack(recorder, postAttnNormStats.invRmsBuffer, decodeBuffers);
292
+ }
293
+ if (releasePostAttnNormWeight && postAttnNormWeightBuf) {
294
+ releaseOrTrack(recorder, postAttnNormWeightBuf, decodeBuffers);
295
+ }
213
296
  releaseOrTrack(recorder, postAttn.buffer, decodeBuffers);
214
297
  if (prenormSumBuffer) {
215
298
  releaseOrTrack(recorder, prenormSumBuffer, decodeBuffers);
@@ -151,6 +151,7 @@ export function summarizeExecutionPlan(plan) {
151
151
  activationDtype: plan.activationDtype,
152
152
  readbackInterval: plan.readbackInterval ?? null,
153
153
  readbackMode: plan.readbackMode ?? null,
154
+ maxBatchDecodeTokens: plan.maxBatchDecodeTokens ?? null,
154
155
  batchSize: plan.defaultBatchSize,
155
156
  stopCheckMode: plan.defaultStopCheckMode,
156
157
  disableCommandBatching: plan.defaultDisableCommandBatching === true,
@@ -162,6 +162,8 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
162
162
  phase: isDecodeMode ? 'decode' : 'prefill',
163
163
  operatorDiagnostics: state.operatorDiagnostics,
164
164
  useFusedGateUpGelu: state.runtimeConfig?.inference?.session?.useFusedGateUpGelu === true,
165
+ useLargeBatchF16F32FusedGateUp: state.runtimeConfig?.inference?.session?.useLargeBatchF16F32FusedGateUp === true,
166
+ skipKVCacheWrites: false,
165
167
  };
166
168
  }
167
169
 
@@ -275,6 +275,7 @@ export function resolvePrefillOptions(state, options = {}) {
275
275
  debug: resolveConfiguredValue(options.debug, state.debug, 'options.debug', (value) => typeof value === 'boolean'),
276
276
  debugLayers: options.debugLayers,
277
277
  profile: resolveConfiguredValue(options.profile, generationDefaults.profile, 'options.profile'),
278
+ benchmark: resolveConfiguredValue(options.benchmark, generationDefaults.benchmark, 'options.benchmark'),
278
279
  disableCommandBatching: executionPlan.disableCommandBatching,
279
280
  disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
280
281
  executionPlan,
@@ -540,7 +540,7 @@ function createDecodeRecorder(state, opts) {
540
540
  if (recorderEnabled) {
541
541
  recorder = opts.profile
542
542
  ? createProfilingRecorder('decode', device)
543
- : createCommandRecorder('decode', { recordLabels: opts.debug === true }, device);
543
+ : createCommandRecorder('decode', { recordLabels: opts.debug === true || opts.benchmark === true }, device);
544
544
  }
545
545
  if (state.decodeStepCount === 1) {
546
546
  const path = selectRuleValue('inference', 'config', 'tracePath', { useRecorder: Boolean(recorder) });
@@ -1323,7 +1323,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1323
1323
  const tokensPerInterval = batchSize * readbackInterval;
1324
1324
  const recorder = opts.profile
1325
1325
  ? createProfilingRecorder('batch_decode', device)
1326
- : createCommandRecorder('batch_decode', { recordLabels: opts.debug === true }, device);
1326
+ : createCommandRecorder('batch_decode', { recordLabels: opts.debug === true || opts.benchmark === true }, device);
1327
1327
  const lmHead = state.weights.get('lm_head');
1328
1328
  if (lmHead && isCpuWeightBuffer(lmHead)) {
1329
1329
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident LM head.');