doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -104,6 +104,39 @@ function getSplitQKVRMSNormRoPEQKRunner(recorder) {
104
104
  );
105
105
  }
106
106
 
107
+ async function projectFusedQGate({
108
+ runMatmulForMode,
109
+ projectionInput,
110
+ layerWeights,
111
+ numTokens,
112
+ qSize,
113
+ hiddenSize,
114
+ layerIdx,
115
+ kernelPath,
116
+ projectionOutputDtype,
117
+ matmulDebug,
118
+ executionPolicies,
119
+ fusedNormWeight,
120
+ fusedNormEps,
121
+ fusedNormOffset,
122
+ }) {
123
+ if (!layerWeights?.qGateProj) {
124
+ return null;
125
+ }
126
+ return runMatmulForMode(projectionInput, layerWeights.qGateProj, numTokens, qSize, hiddenSize, {
127
+ transposeB: 'auto',
128
+ role: 'q_proj',
129
+ layerIdx,
130
+ kernelPath,
131
+ outputDtype: projectionOutputDtype,
132
+ matmulDebug,
133
+ executionPolicies,
134
+ normWeight: fusedNormWeight,
135
+ rmsNormEps: fusedNormEps,
136
+ rmsNormOffset: fusedNormOffset,
137
+ });
138
+ }
139
+
107
140
  function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemporary) {
108
141
  if (isGpuBufferInstance(layerWeight) || isWeightBuffer(layerWeight)) {
109
142
  return;
@@ -564,6 +597,7 @@ export async function projectAttentionQKV({
564
597
  }) {
565
598
  const runMatmulForMode = getMatmulRunner(recorder);
566
599
  const runSplitForMode = getSplitRunner(recorder);
600
+ const runSplitQGForMode = getSplitQGRunner(recorder);
567
601
  const runSplitQKVRMSNormQKForMode = getSplitQKVRMSNormQKRunner(recorder);
568
602
  const runSplitQKVRMSNormRoPEQKForMode = getSplitQKVRMSNormRoPEQKRunner(recorder);
569
603
  const reuseSharedKV = sharedKTensor != null || sharedVTensor != null;
@@ -605,6 +639,21 @@ export async function projectAttentionQKV({
605
639
  if (useFusedQKV && layerWeights.qkvProj && layerWeights.qkvSizes) {
606
640
  const [qSizeFused, kSizeFused, vSizeFused] = layerWeights.qkvSizes;
607
641
  const qkvSizeTotal = qSizeFused + kSizeFused + vSizeFused;
642
+ const qProjectionSize = numHeads * headDim;
643
+ const qProjectionContainsGate = attentionOutputGate === true;
644
+ const hasSeparateGateProjection = qProjectionContainsGate && layerWeights.qGateProj;
645
+ if (hasSeparateGateProjection && qSizeFused !== qProjectionSize) {
646
+ throw new Error(
647
+ `Fused QKV for attention-output-gate layer ${layerIdx} with qGateProj must store ` +
648
+ `Q-only first slice with ${qProjectionSize} columns; got ${qSizeFused}.`
649
+ );
650
+ }
651
+ if (qProjectionContainsGate && !hasSeparateGateProjection && qSizeFused !== qProjectionSize * 2) {
652
+ throw new Error(
653
+ `Fused QKV for attention-output-gate layer ${layerIdx} must store Q+gate first slice ` +
654
+ `with ${qProjectionSize * 2} columns; got ${qSizeFused}.`
655
+ );
656
+ }
608
657
  let qkvTensor = null;
609
658
  let qNormBuf = null;
610
659
  let kNormBuf = null;
@@ -629,6 +678,7 @@ export async function projectAttentionQKV({
629
678
  ? { ...qkNormRoPEFusion, headDim }
630
679
  : null;
631
680
  const canFuseSplitQKNormAndRoPE = qkNormRoPEFusion?.enabled === true
681
+ && (!qProjectionContainsGate || hasSeparateGateProjection)
632
682
  && qkNormRoPEFusion.projectionDiagnosticsEnabled !== true
633
683
  && qkNormRoPEFusion.skipKNorm !== true
634
684
  && qkNormRoPEFusion.allowUnitQKNorm !== true
@@ -669,12 +719,44 @@ export async function projectAttentionQKV({
669
719
  );
670
720
  releaseTemporary(qkvTensor.buffer);
671
721
  qkvTensor = null;
722
+ let qGateTensor = null;
723
+ if (hasSeparateGateProjection) {
724
+ try {
725
+ qGateTensor = await projectFusedQGate({
726
+ runMatmulForMode,
727
+ projectionInput,
728
+ layerWeights,
729
+ numTokens,
730
+ qSize: qProjectionSize,
731
+ hiddenSize,
732
+ layerIdx,
733
+ kernelPath,
734
+ projectionOutputDtype,
735
+ matmulDebug,
736
+ executionPolicies,
737
+ fusedNormWeight,
738
+ fusedNormEps,
739
+ fusedNormOffset,
740
+ });
741
+ } catch (error) {
742
+ if (fused.Q?.buffer) {
743
+ releaseTemporary(fused.Q.buffer);
744
+ }
745
+ if (fused.K?.buffer) {
746
+ releaseTemporary(fused.K.buffer);
747
+ }
748
+ if (fused.V?.buffer) {
749
+ releaseTemporary(fused.V.buffer);
750
+ }
751
+ throw error;
752
+ }
753
+ }
672
754
  if (onFusedQKV) {
673
755
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
674
756
  }
675
757
  return {
676
758
  qTensor: fused.Q,
677
- qGateTensor: null,
759
+ qGateTensor,
678
760
  kTensor: fused.K,
679
761
  vTensor: fused.V,
680
762
  usedFusedQKV: true,
@@ -686,6 +768,7 @@ export async function projectAttentionQKV({
686
768
  }
687
769
  }
688
770
  const canFuseSplitAndQKNorm = qkNormFusion?.enabled === true
771
+ && (!qProjectionContainsGate || hasSeparateGateProjection)
689
772
  && qkNormFusion.projectionDiagnosticsEnabled !== true
690
773
  && qkNormFusion.skipKNorm !== true
691
774
  && qkNormFusion.allowUnitQKNorm !== true
@@ -717,12 +800,44 @@ export async function projectAttentionQKV({
717
800
  );
718
801
  releaseTemporary(qkvTensor.buffer);
719
802
  qkvTensor = null;
803
+ let qGateTensor = null;
804
+ if (hasSeparateGateProjection) {
805
+ try {
806
+ qGateTensor = await projectFusedQGate({
807
+ runMatmulForMode,
808
+ projectionInput,
809
+ layerWeights,
810
+ numTokens,
811
+ qSize: qProjectionSize,
812
+ hiddenSize,
813
+ layerIdx,
814
+ kernelPath,
815
+ projectionOutputDtype,
816
+ matmulDebug,
817
+ executionPolicies,
818
+ fusedNormWeight,
819
+ fusedNormEps,
820
+ fusedNormOffset,
821
+ });
822
+ } catch (error) {
823
+ if (fused.Q?.buffer) {
824
+ releaseTemporary(fused.Q.buffer);
825
+ }
826
+ if (fused.K?.buffer) {
827
+ releaseTemporary(fused.K.buffer);
828
+ }
829
+ if (fused.V?.buffer) {
830
+ releaseTemporary(fused.V.buffer);
831
+ }
832
+ throw error;
833
+ }
834
+ }
720
835
  if (onFusedQKV) {
721
836
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
722
837
  }
723
838
  return {
724
839
  qTensor: fused.Q,
725
- qGateTensor: null,
840
+ qGateTensor,
726
841
  kTensor: fused.K,
727
842
  vTensor: fused.V,
728
843
  usedFusedQKV: true,
@@ -740,12 +855,67 @@ export async function projectAttentionQKV({
740
855
  vSize: vSizeFused,
741
856
  });
742
857
  releaseTemporary(qkvTensor.buffer);
858
+ qkvTensor = null;
859
+ let qTensor = split.Q;
860
+ let qGateTensor = null;
861
+ if (hasSeparateGateProjection) {
862
+ try {
863
+ qGateTensor = await projectFusedQGate({
864
+ runMatmulForMode,
865
+ projectionInput,
866
+ layerWeights,
867
+ numTokens,
868
+ qSize: qProjectionSize,
869
+ hiddenSize,
870
+ layerIdx,
871
+ kernelPath,
872
+ projectionOutputDtype,
873
+ matmulDebug,
874
+ executionPolicies,
875
+ fusedNormWeight,
876
+ fusedNormEps,
877
+ fusedNormOffset,
878
+ });
879
+ } catch (error) {
880
+ if (split.Q?.buffer) {
881
+ releaseTemporary(split.Q.buffer);
882
+ }
883
+ if (split.K?.buffer) {
884
+ releaseTemporary(split.K.buffer);
885
+ }
886
+ if (split.V?.buffer) {
887
+ releaseTemporary(split.V.buffer);
888
+ }
889
+ throw error;
890
+ }
891
+ } else if (qProjectionContainsGate) {
892
+ const qgTensor = split.Q;
893
+ try {
894
+ const qg = await runSplitQGForMode(qgTensor, {
895
+ numTokens,
896
+ numHeads,
897
+ headDim,
898
+ });
899
+ qTensor = qg.Q;
900
+ qGateTensor = qg.G;
901
+ releaseTemporary(qgTensor.buffer);
902
+ } catch (error) {
903
+ releaseTemporary(qgTensor.buffer);
904
+ if (split.K?.buffer) {
905
+ releaseTemporary(split.K.buffer);
906
+ }
907
+ if (split.V?.buffer) {
908
+ releaseTemporary(split.V.buffer);
909
+ }
910
+ throw error;
911
+ }
912
+ }
743
913
  if (onFusedQKV) {
744
914
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
745
915
  }
746
916
  return {
747
- qTensor: split.Q,
748
- qGateTensor: null,
917
+ qTensor,
918
+ qGateTensor,
749
919
  kTensor: split.K,
750
920
  vTensor: split.V,
751
921
  usedFusedQKV: true,
@@ -62,6 +62,7 @@ import {
62
62
  isAttentionKvDtypeExplicit,
63
63
  } from './precision-contract.js';
64
64
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
65
+ import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
65
66
 
66
67
  const ATTENTION_DTYPE_LOGGED = new Set();
67
68
 
@@ -390,20 +391,23 @@ export async function recordLayerAttentionGPU(
390
391
  const runtimeSession = getRuntimeConfig()?.inference?.session;
391
392
  const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
392
393
  const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
393
- const directF16KVCacheWrite = resolveDirectF16KVCacheWrite({
394
- state,
395
- layerIdx,
396
- currentSeqLen,
397
- numTokens,
398
- numKVHeads,
399
- headDim,
400
- reusesSharedKV,
401
- storeSharedKV,
402
- diffusionGemmaDecoder,
403
- valueNorm: config.valueNorm === true,
404
- diagnosticsEnabled: qkNormRoPEDiagnostics,
405
- disableRoPE,
406
- });
394
+ const skipKVCacheWrites = state.skipKVCacheWrites === true;
395
+ const directF16KVCacheWrite = skipKVCacheWrites
396
+ ? null
397
+ : resolveDirectF16KVCacheWrite({
398
+ state,
399
+ layerIdx,
400
+ currentSeqLen,
401
+ numTokens,
402
+ numKVHeads,
403
+ headDim,
404
+ reusesSharedKV,
405
+ storeSharedKV,
406
+ diffusionGemmaDecoder,
407
+ valueNorm: config.valueNorm === true,
408
+ diagnosticsEnabled: qkNormRoPEDiagnostics,
409
+ disableRoPE,
410
+ });
407
411
  let kvCacheWriteFused = false;
408
412
  ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
409
413
  recorder,
@@ -649,7 +653,7 @@ export async function recordLayerAttentionGPU(
649
653
  }
650
654
 
651
655
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
652
- if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
656
+ if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
653
657
  if (kvCacheWriteFused) {
654
658
  state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
655
659
  } else if (state.kvCache.kvDtype === 'f16') {
@@ -741,7 +745,9 @@ export async function recordLayerAttentionGPU(
741
745
  }
742
746
  decoderKVState = kvState;
743
747
  } else {
744
- kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
748
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
749
+ forceCurrentTensors: skipKVCacheWrites,
750
+ });
745
751
  }
746
752
  const dispatchConfig = {
747
753
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
@@ -758,6 +764,7 @@ export async function recordLayerAttentionGPU(
758
764
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
759
765
  prefillFallbackNeedsCast, causalForAttention,
760
766
  } = dispatchParams;
767
+ const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
761
768
 
762
769
  // 5. Attention
763
770
 
@@ -767,6 +774,7 @@ export async function recordLayerAttentionGPU(
767
774
  usedFusedQKV, qTensor, kTensor, vTensor,
768
775
  ));
769
776
  const mergedSessionRec = getRuntimeConfig()?.inference?.session;
777
+ let attentionOutputGateFused = false;
770
778
 
771
779
  const attentionKernelRunners = {
772
780
  bdpa: async () => {
@@ -923,7 +931,18 @@ export async function recordLayerAttentionGPU(
923
931
  // path in ./run.js.
924
932
  const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
925
933
  const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
926
- return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
934
+ const useOutputGateFusion = canUseAttentionOutputGateFusion({
935
+ session: mergedSessionRec,
936
+ qGateTensor,
937
+ numTokens,
938
+ numHeads,
939
+ headDim,
940
+ cachedKDtype,
941
+ cachedVDtype,
942
+ kernelPath: attentionKernelPath,
943
+ diffusionGemmaDecoder,
944
+ });
945
+ const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
927
946
  seqLen: numTokens,
928
947
  kvLen: kvState.kvLenForAttention,
929
948
  numKVHeads,
@@ -939,10 +958,13 @@ export async function recordLayerAttentionGPU(
939
958
  kvLayout: kvState.kvLayout,
940
959
  kvPageTable: kvState.kvPageTable,
941
960
  kvPageSize: kvState.kvPageSize,
942
- kernelPath,
943
- useFlashPrefill: useFlashPrefillRec,
944
- useOrtFlashPrefill: useOrtFlashPrefillRec,
961
+ kernelPath: attentionKernelPath,
962
+ useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
963
+ useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
964
+ outputGate: useOutputGateFusion ? qGateTensor : null,
945
965
  });
966
+ attentionOutputGateFused = result?.outputGateFused === true;
967
+ return result;
946
968
  },
947
969
  };
948
970
  const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
@@ -980,13 +1002,14 @@ export async function recordLayerAttentionGPU(
980
1002
  }
981
1003
 
982
1004
  attnForProjection = attnOutput;
983
- if (qGateTensor) {
1005
+ if (qGateTensor && !attentionOutputGateFused) {
984
1006
  // Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
985
1007
  // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
986
1008
  const gateActivation = 'sigmoid';
987
1009
  attnForProjection = await recordSiLU(recorder, attnOutput, {
988
1010
  size: numTokens * numHeads * headDim,
989
1011
  gate: qGateTensor,
1012
+ useVec4: (numTokens * numHeads * headDim) % 4 === 0,
990
1013
  gateActivation,
991
1014
  inputActivation: 'identity',
992
1015
  swigluLimit: null,
@@ -66,6 +66,7 @@ import {
66
66
  isAttentionKvDtypeExplicit,
67
67
  } from './precision-contract.js';
68
68
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
69
+ import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
69
70
 
70
71
  const ATTENTION_DTYPE_LOGGED = new Set();
71
72
 
@@ -793,6 +794,8 @@ export async function runLayerAttentionGPU(
793
794
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
794
795
  prefillFallbackNeedsCast, causalForAttention,
795
796
  } = dispatchParams;
797
+ const mergedSession = getRuntimeConfig()?.inference?.session;
798
+ let attentionOutputGateFused = false;
796
799
 
797
800
  // 5. Attention (uses raw GPUBuffers)
798
801
 
@@ -991,9 +994,19 @@ export async function runLayerAttentionGPU(
991
994
  // session (state.runtimeConfig.inference.session is a stale snapshot —
992
995
  // see comment above at the rmsNorm fusion check). The kernel itself
993
996
  // enforces remaining preconditions (head_dim=256, etc.).
994
- const mergedSession = getRuntimeConfig()?.inference?.session;
995
997
  const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
996
998
  const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
999
+ const useOutputGateFusion = canUseAttentionOutputGateFusion({
1000
+ session: mergedSession,
1001
+ qGateTensor,
1002
+ numTokens,
1003
+ numHeads,
1004
+ headDim,
1005
+ cachedKDtype,
1006
+ cachedVDtype,
1007
+ kernelPath,
1008
+ diffusionGemmaDecoder,
1009
+ });
997
1010
  const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
998
1011
  seqLen: numTokens,
999
1012
  kvLen: kvState.kvLenForAttention,
@@ -1013,7 +1026,9 @@ export async function runLayerAttentionGPU(
1013
1026
  kernelPath,
1014
1027
  useFlashPrefill,
1015
1028
  useOrtFlashPrefill,
1029
+ outputGate: useOutputGateFusion ? qGateTensor : null,
1016
1030
  });
1031
+ attentionOutputGateFused = result?.outputGateFused === true;
1017
1032
  if (prefillFallbackNeedsCast) {
1018
1033
  if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
1019
1034
  if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
@@ -1068,7 +1083,7 @@ export async function runLayerAttentionGPU(
1068
1083
  }
1069
1084
 
1070
1085
  attnForProjection = attnOutput;
1071
- if (qGateTensor) {
1086
+ if (qGateTensor && !attentionOutputGateFused) {
1072
1087
  // The Qwen3_5 reference (HF transformers) full-attention path
1073
1088
  // applies sigmoid(gate) — NOT silu(gate) — to the attention
1074
1089
  // output regardless of how the architecture's gate label is
@@ -1089,6 +1104,7 @@ export async function runLayerAttentionGPU(
1089
1104
  attnForProjection = await runSiLU(attnOutput, {
1090
1105
  size: numTokens * numHeads * headDim,
1091
1106
  gate: qGateTensor,
1107
+ useVec4: (numTokens * numHeads * headDim) % 4 === 0,
1092
1108
  gateActivation,
1093
1109
  inputActivation: 'identity',
1094
1110
  swigluLimit: null,
@@ -1125,7 +1141,6 @@ export async function runLayerAttentionGPU(
1125
1141
  // Use fused o_proj + residual for decode when possible
1126
1142
  // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
1127
1143
  const oProjDtype = getWeightDtype(oProjBuf);
1128
- const mergedSession = getRuntimeConfig()?.inference?.session;
1129
1144
  const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
1130
1145
  allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
1131
1146
  hasResidual: Boolean(residualTensor),
@@ -114,14 +114,32 @@ export interface TensorInfo {
114
114
  export interface ParsedSessionSettings {
115
115
  prefillChunkSubmitMode: 'sync' | 'async' | null;
116
116
  prefillTokenChunkSize: number | null;
117
+ skipEmbeddingKVCacheWrites: boolean | null;
117
118
  useFlashPrefillAttention: boolean | null;
119
+ useLargeBatchF16F32FusedGateUp: boolean | null;
118
120
  useWideTileQ4KPrefill: boolean | null;
119
121
  useWideTileQ4KDecode: boolean | null;
120
122
  useSandwichRMSNormPairFusion: boolean | null;
121
123
  usePostFfnNextInputRMSNormPairFusion: boolean | null;
124
+ usePostAttnNormFusedGateUp: boolean | null;
125
+ useLinearAttentionABProjectionFusion: boolean | null;
126
+ useLinearAttentionQKVZProjectionFusion: boolean | null;
127
+ useLinearAttentionFusedDecodeCore: boolean | null;
128
+ useWideTileResidualFusion: boolean | null;
129
+ useFusedRmsnormWideTile: boolean | null;
122
130
  useFusedQKVSplitQKNorm: boolean | null;
123
131
  useFusedQKVSplitQKNormRoPE: boolean | null;
124
132
  retainQ4KMaterialization: boolean | null;
133
+ lmHeadArgmaxQ4K: {
134
+ useFullBlockFastPath?: boolean;
135
+ colsPerWorkgroup?: number;
136
+ threadsPerCol?: number;
137
+ } | null;
138
+ attentionDecodeOnline: {
139
+ workgroupSize?: 128 | 256;
140
+ useDirectContiguousKVLayout?: boolean;
141
+ useOutputGateFusion?: boolean;
142
+ } | null;
125
143
  }
126
144
 
127
145
  export type Manifest = RuntimeModelContract;
@@ -223,10 +223,22 @@ function resolveSessionSettings(inferenceConfig, modelId) {
223
223
  `"${String(tokenChunk)}"; expected null or a positive integer.`
224
224
  );
225
225
  }
226
+ const skipEmbeddingKVCacheWrites = session?.skipEmbeddingKVCacheWrites;
227
+ if (
228
+ skipEmbeddingKVCacheWrites !== undefined
229
+ && skipEmbeddingKVCacheWrites !== null
230
+ && typeof skipEmbeddingKVCacheWrites !== 'boolean'
231
+ ) {
232
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.skipEmbeddingKVCacheWrites "${String(skipEmbeddingKVCacheWrites)}"; expected boolean.`);
233
+ }
226
234
  const flash = session?.useFlashPrefillAttention;
227
235
  if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
228
236
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
229
237
  }
238
+ const largeBatchF16F32FusedGateUp = session?.useLargeBatchF16F32FusedGateUp;
239
+ if (largeBatchF16F32FusedGateUp !== undefined && largeBatchF16F32FusedGateUp !== null && typeof largeBatchF16F32FusedGateUp !== 'boolean') {
240
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLargeBatchF16F32FusedGateUp "${String(largeBatchF16F32FusedGateUp)}"; expected boolean.`);
241
+ }
230
242
  const wide = session?.useWideTileQ4KPrefill;
231
243
  if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
232
244
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
@@ -243,6 +255,30 @@ function resolveSessionSettings(inferenceConfig, modelId) {
243
255
  if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
244
256
  throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
245
257
  }
258
+ const postAttnNormFusedGateUp = session?.usePostAttnNormFusedGateUp;
259
+ if (postAttnNormFusedGateUp !== undefined && postAttnNormFusedGateUp !== null && typeof postAttnNormFusedGateUp !== 'boolean') {
260
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostAttnNormFusedGateUp "${String(postAttnNormFusedGateUp)}"; expected boolean.`);
261
+ }
262
+ const linearAttentionABProjectionFusion = session?.useLinearAttentionABProjectionFusion;
263
+ if (linearAttentionABProjectionFusion !== undefined && linearAttentionABProjectionFusion !== null && typeof linearAttentionABProjectionFusion !== 'boolean') {
264
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionABProjectionFusion "${String(linearAttentionABProjectionFusion)}"; expected boolean.`);
265
+ }
266
+ const linearAttentionQKVZProjectionFusion = session?.useLinearAttentionQKVZProjectionFusion;
267
+ if (linearAttentionQKVZProjectionFusion !== undefined && linearAttentionQKVZProjectionFusion !== null && typeof linearAttentionQKVZProjectionFusion !== 'boolean') {
268
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionQKVZProjectionFusion "${String(linearAttentionQKVZProjectionFusion)}"; expected boolean.`);
269
+ }
270
+ const linearAttentionFusedDecodeCore = session?.useLinearAttentionFusedDecodeCore;
271
+ if (linearAttentionFusedDecodeCore !== undefined && linearAttentionFusedDecodeCore !== null && typeof linearAttentionFusedDecodeCore !== 'boolean') {
272
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionFusedDecodeCore "${String(linearAttentionFusedDecodeCore)}"; expected boolean.`);
273
+ }
274
+ const wideTileResidualFusion = session?.useWideTileResidualFusion;
275
+ if (wideTileResidualFusion !== undefined && wideTileResidualFusion !== null && typeof wideTileResidualFusion !== 'boolean') {
276
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileResidualFusion "${String(wideTileResidualFusion)}"; expected boolean.`);
277
+ }
278
+ const fusedRmsnormWideTile = session?.useFusedRmsnormWideTile;
279
+ if (fusedRmsnormWideTile !== undefined && fusedRmsnormWideTile !== null && typeof fusedRmsnormWideTile !== 'boolean') {
280
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedRmsnormWideTile "${String(fusedRmsnormWideTile)}"; expected boolean.`);
281
+ }
246
282
  const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
247
283
  if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
248
284
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
@@ -255,17 +291,81 @@ function resolveSessionSettings(inferenceConfig, modelId) {
255
291
  if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
256
292
  throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
257
293
  }
294
+ const lmHeadArgmaxQ4K = session?.lmHeadArgmaxQ4K;
295
+ if (lmHeadArgmaxQ4K !== undefined && lmHeadArgmaxQ4K !== null) {
296
+ if (typeof lmHeadArgmaxQ4K !== 'object' || Array.isArray(lmHeadArgmaxQ4K)) {
297
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K; expected object or null.`);
298
+ }
299
+ const fullBlockFastPath = lmHeadArgmaxQ4K.useFullBlockFastPath;
300
+ if (fullBlockFastPath !== undefined && typeof fullBlockFastPath !== 'boolean') {
301
+ throw new Error(
302
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.useFullBlockFastPath ` +
303
+ `"${String(fullBlockFastPath)}"; expected boolean.`
304
+ );
305
+ }
306
+ const colsPerWorkgroup = lmHeadArgmaxQ4K.colsPerWorkgroup;
307
+ if (colsPerWorkgroup !== undefined && (!Number.isInteger(colsPerWorkgroup) || colsPerWorkgroup <= 0)) {
308
+ throw new Error(
309
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.colsPerWorkgroup ` +
310
+ `"${String(colsPerWorkgroup)}"; expected positive integer.`
311
+ );
312
+ }
313
+ const threadsPerCol = lmHeadArgmaxQ4K.threadsPerCol;
314
+ if (threadsPerCol !== undefined && (!Number.isInteger(threadsPerCol) || threadsPerCol <= 0)) {
315
+ throw new Error(
316
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.threadsPerCol ` +
317
+ `"${String(threadsPerCol)}"; expected positive integer.`
318
+ );
319
+ }
320
+ }
321
+ const attentionDecodeOnline = session?.attentionDecodeOnline;
322
+ if (attentionDecodeOnline !== undefined && attentionDecodeOnline !== null) {
323
+ if (typeof attentionDecodeOnline !== 'object' || Array.isArray(attentionDecodeOnline)) {
324
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline; expected object or null.`);
325
+ }
326
+ const workgroupSize = attentionDecodeOnline.workgroupSize;
327
+ if (workgroupSize !== undefined && workgroupSize !== 128 && workgroupSize !== 256) {
328
+ throw new Error(
329
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.workgroupSize ` +
330
+ `"${String(workgroupSize)}"; expected 128 or 256.`
331
+ );
332
+ }
333
+ const directKv = attentionDecodeOnline.useDirectContiguousKVLayout;
334
+ if (directKv !== undefined && typeof directKv !== 'boolean') {
335
+ throw new Error(
336
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useDirectContiguousKVLayout ` +
337
+ `"${String(directKv)}"; expected boolean.`
338
+ );
339
+ }
340
+ const outputGateFusion = attentionDecodeOnline.useOutputGateFusion;
341
+ if (outputGateFusion !== undefined && typeof outputGateFusion !== 'boolean') {
342
+ throw new Error(
343
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useOutputGateFusion ` +
344
+ `"${String(outputGateFusion)}"; expected boolean.`
345
+ );
346
+ }
347
+ }
258
348
  return {
259
349
  prefillChunkSubmitMode: submit ?? null,
260
350
  prefillTokenChunkSize: tokenChunk ?? null,
351
+ skipEmbeddingKVCacheWrites: skipEmbeddingKVCacheWrites ?? null,
261
352
  useFlashPrefillAttention: flash ?? null,
353
+ useLargeBatchF16F32FusedGateUp: largeBatchF16F32FusedGateUp ?? null,
262
354
  useWideTileQ4KPrefill: wide ?? null,
263
355
  useWideTileQ4KDecode: wideDecode ?? null,
264
356
  useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
265
357
  usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
358
+ usePostAttnNormFusedGateUp: postAttnNormFusedGateUp ?? null,
359
+ useLinearAttentionABProjectionFusion: linearAttentionABProjectionFusion ?? null,
360
+ useLinearAttentionQKVZProjectionFusion: linearAttentionQKVZProjectionFusion ?? null,
361
+ useLinearAttentionFusedDecodeCore: linearAttentionFusedDecodeCore ?? null,
362
+ useWideTileResidualFusion: wideTileResidualFusion ?? null,
363
+ useFusedRmsnormWideTile: fusedRmsnormWideTile ?? null,
266
364
  useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
267
365
  useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
268
366
  retainQ4KMaterialization: retain ?? null,
367
+ lmHeadArgmaxQ4K: lmHeadArgmaxQ4K ?? null,
368
+ attentionDecodeOnline: attentionDecodeOnline ?? null,
269
369
  };
270
370
  }
271
371
 
@@ -21,6 +21,7 @@ export interface StaticExecutionPlan {
21
21
  defaultMaxTokens: number;
22
22
  readbackInterval: number | null;
23
23
  readbackMode: 'sequential' | 'overlapped' | 'auto';
24
+ maxBatchDecodeTokens?: number | null;
24
25
  ringTokens: number | null;
25
26
  ringStop: number | null;
26
27
  ringStaging: number | null;
@@ -53,6 +54,7 @@ export interface ExecutionSessionPlan {
53
54
  maxTokens: number;
54
55
  readbackInterval: number | null;
55
56
  readbackMode: 'sequential' | 'overlapped' | 'auto';
57
+ maxBatchDecodeTokens?: number | null;
56
58
  ringTokens: number | null;
57
59
  ringStop: number | null;
58
60
  ringStaging: number | null;