doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -104,6 +104,39 @@ function getSplitQKVRMSNormRoPEQKRunner(recorder) {
104
104
  );
105
105
  }
106
106
 
107
+ async function projectFusedQGate({
108
+ runMatmulForMode,
109
+ projectionInput,
110
+ layerWeights,
111
+ numTokens,
112
+ qSize,
113
+ hiddenSize,
114
+ layerIdx,
115
+ kernelPath,
116
+ projectionOutputDtype,
117
+ matmulDebug,
118
+ executionPolicies,
119
+ fusedNormWeight,
120
+ fusedNormEps,
121
+ fusedNormOffset,
122
+ }) {
123
+ if (!layerWeights?.qGateProj) {
124
+ return null;
125
+ }
126
+ return runMatmulForMode(projectionInput, layerWeights.qGateProj, numTokens, qSize, hiddenSize, {
127
+ transposeB: 'auto',
128
+ role: 'q_proj',
129
+ layerIdx,
130
+ kernelPath,
131
+ outputDtype: projectionOutputDtype,
132
+ matmulDebug,
133
+ executionPolicies,
134
+ normWeight: fusedNormWeight,
135
+ rmsNormEps: fusedNormEps,
136
+ rmsNormOffset: fusedNormOffset,
137
+ });
138
+ }
139
+
107
140
  function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemporary) {
108
141
  if (isGpuBufferInstance(layerWeight) || isWeightBuffer(layerWeight)) {
109
142
  return;
@@ -564,6 +597,7 @@ export async function projectAttentionQKV({
564
597
  }) {
565
598
  const runMatmulForMode = getMatmulRunner(recorder);
566
599
  const runSplitForMode = getSplitRunner(recorder);
600
+ const runSplitQGForMode = getSplitQGRunner(recorder);
567
601
  const runSplitQKVRMSNormQKForMode = getSplitQKVRMSNormQKRunner(recorder);
568
602
  const runSplitQKVRMSNormRoPEQKForMode = getSplitQKVRMSNormRoPEQKRunner(recorder);
569
603
  const reuseSharedKV = sharedKTensor != null || sharedVTensor != null;
@@ -605,6 +639,21 @@ export async function projectAttentionQKV({
605
639
  if (useFusedQKV && layerWeights.qkvProj && layerWeights.qkvSizes) {
606
640
  const [qSizeFused, kSizeFused, vSizeFused] = layerWeights.qkvSizes;
607
641
  const qkvSizeTotal = qSizeFused + kSizeFused + vSizeFused;
642
+ const qProjectionSize = numHeads * headDim;
643
+ const qProjectionContainsGate = attentionOutputGate === true;
644
+ const hasSeparateGateProjection = qProjectionContainsGate && layerWeights.qGateProj;
645
+ if (hasSeparateGateProjection && qSizeFused !== qProjectionSize) {
646
+ throw new Error(
647
+ `Fused QKV for attention-output-gate layer ${layerIdx} with qGateProj must store ` +
648
+ `Q-only first slice with ${qProjectionSize} columns; got ${qSizeFused}.`
649
+ );
650
+ }
651
+ if (qProjectionContainsGate && !hasSeparateGateProjection && qSizeFused !== qProjectionSize * 2) {
652
+ throw new Error(
653
+ `Fused QKV for attention-output-gate layer ${layerIdx} must store Q+gate first slice ` +
654
+ `with ${qProjectionSize * 2} columns; got ${qSizeFused}.`
655
+ );
656
+ }
608
657
  let qkvTensor = null;
609
658
  let qNormBuf = null;
610
659
  let kNormBuf = null;
@@ -629,6 +678,7 @@ export async function projectAttentionQKV({
629
678
  ? { ...qkNormRoPEFusion, headDim }
630
679
  : null;
631
680
  const canFuseSplitQKNormAndRoPE = qkNormRoPEFusion?.enabled === true
681
+ && (!qProjectionContainsGate || hasSeparateGateProjection)
632
682
  && qkNormRoPEFusion.projectionDiagnosticsEnabled !== true
633
683
  && qkNormRoPEFusion.skipKNorm !== true
634
684
  && qkNormRoPEFusion.allowUnitQKNorm !== true
@@ -669,12 +719,44 @@ export async function projectAttentionQKV({
669
719
  );
670
720
  releaseTemporary(qkvTensor.buffer);
671
721
  qkvTensor = null;
722
+ let qGateTensor = null;
723
+ if (hasSeparateGateProjection) {
724
+ try {
725
+ qGateTensor = await projectFusedQGate({
726
+ runMatmulForMode,
727
+ projectionInput,
728
+ layerWeights,
729
+ numTokens,
730
+ qSize: qProjectionSize,
731
+ hiddenSize,
732
+ layerIdx,
733
+ kernelPath,
734
+ projectionOutputDtype,
735
+ matmulDebug,
736
+ executionPolicies,
737
+ fusedNormWeight,
738
+ fusedNormEps,
739
+ fusedNormOffset,
740
+ });
741
+ } catch (error) {
742
+ if (fused.Q?.buffer) {
743
+ releaseTemporary(fused.Q.buffer);
744
+ }
745
+ if (fused.K?.buffer) {
746
+ releaseTemporary(fused.K.buffer);
747
+ }
748
+ if (fused.V?.buffer) {
749
+ releaseTemporary(fused.V.buffer);
750
+ }
751
+ throw error;
752
+ }
753
+ }
672
754
  if (onFusedQKV) {
673
755
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
674
756
  }
675
757
  return {
676
758
  qTensor: fused.Q,
677
- qGateTensor: null,
759
+ qGateTensor,
678
760
  kTensor: fused.K,
679
761
  vTensor: fused.V,
680
762
  usedFusedQKV: true,
@@ -686,6 +768,7 @@ export async function projectAttentionQKV({
686
768
  }
687
769
  }
688
770
  const canFuseSplitAndQKNorm = qkNormFusion?.enabled === true
771
+ && (!qProjectionContainsGate || hasSeparateGateProjection)
689
772
  && qkNormFusion.projectionDiagnosticsEnabled !== true
690
773
  && qkNormFusion.skipKNorm !== true
691
774
  && qkNormFusion.allowUnitQKNorm !== true
@@ -717,12 +800,44 @@ export async function projectAttentionQKV({
717
800
  );
718
801
  releaseTemporary(qkvTensor.buffer);
719
802
  qkvTensor = null;
803
+ let qGateTensor = null;
804
+ if (hasSeparateGateProjection) {
805
+ try {
806
+ qGateTensor = await projectFusedQGate({
807
+ runMatmulForMode,
808
+ projectionInput,
809
+ layerWeights,
810
+ numTokens,
811
+ qSize: qProjectionSize,
812
+ hiddenSize,
813
+ layerIdx,
814
+ kernelPath,
815
+ projectionOutputDtype,
816
+ matmulDebug,
817
+ executionPolicies,
818
+ fusedNormWeight,
819
+ fusedNormEps,
820
+ fusedNormOffset,
821
+ });
822
+ } catch (error) {
823
+ if (fused.Q?.buffer) {
824
+ releaseTemporary(fused.Q.buffer);
825
+ }
826
+ if (fused.K?.buffer) {
827
+ releaseTemporary(fused.K.buffer);
828
+ }
829
+ if (fused.V?.buffer) {
830
+ releaseTemporary(fused.V.buffer);
831
+ }
832
+ throw error;
833
+ }
834
+ }
720
835
  if (onFusedQKV) {
721
836
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
722
837
  }
723
838
  return {
724
839
  qTensor: fused.Q,
725
- qGateTensor: null,
840
+ qGateTensor,
726
841
  kTensor: fused.K,
727
842
  vTensor: fused.V,
728
843
  usedFusedQKV: true,
@@ -740,12 +855,67 @@ export async function projectAttentionQKV({
740
855
  vSize: vSizeFused,
741
856
  });
742
857
  releaseTemporary(qkvTensor.buffer);
858
+ qkvTensor = null;
859
+ let qTensor = split.Q;
860
+ let qGateTensor = null;
861
+ if (hasSeparateGateProjection) {
862
+ try {
863
+ qGateTensor = await projectFusedQGate({
864
+ runMatmulForMode,
865
+ projectionInput,
866
+ layerWeights,
867
+ numTokens,
868
+ qSize: qProjectionSize,
869
+ hiddenSize,
870
+ layerIdx,
871
+ kernelPath,
872
+ projectionOutputDtype,
873
+ matmulDebug,
874
+ executionPolicies,
875
+ fusedNormWeight,
876
+ fusedNormEps,
877
+ fusedNormOffset,
878
+ });
879
+ } catch (error) {
880
+ if (split.Q?.buffer) {
881
+ releaseTemporary(split.Q.buffer);
882
+ }
883
+ if (split.K?.buffer) {
884
+ releaseTemporary(split.K.buffer);
885
+ }
886
+ if (split.V?.buffer) {
887
+ releaseTemporary(split.V.buffer);
888
+ }
889
+ throw error;
890
+ }
891
+ } else if (qProjectionContainsGate) {
892
+ const qgTensor = split.Q;
893
+ try {
894
+ const qg = await runSplitQGForMode(qgTensor, {
895
+ numTokens,
896
+ numHeads,
897
+ headDim,
898
+ });
899
+ qTensor = qg.Q;
900
+ qGateTensor = qg.G;
901
+ releaseTemporary(qgTensor.buffer);
902
+ } catch (error) {
903
+ releaseTemporary(qgTensor.buffer);
904
+ if (split.K?.buffer) {
905
+ releaseTemporary(split.K.buffer);
906
+ }
907
+ if (split.V?.buffer) {
908
+ releaseTemporary(split.V.buffer);
909
+ }
910
+ throw error;
911
+ }
912
+ }
743
913
  if (onFusedQKV) {
744
914
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
745
915
  }
746
916
  return {
747
- qTensor: split.Q,
748
- qGateTensor: null,
917
+ qTensor,
918
+ qGateTensor,
749
919
  kTensor: split.K,
750
920
  vTensor: split.V,
751
921
  usedFusedQKV: true,
@@ -62,6 +62,7 @@ import {
62
62
  isAttentionKvDtypeExplicit,
63
63
  } from './precision-contract.js';
64
64
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
65
+ import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
65
66
 
66
67
  const ATTENTION_DTYPE_LOGGED = new Set();
67
68
 
@@ -390,20 +391,23 @@ export async function recordLayerAttentionGPU(
390
391
  const runtimeSession = getRuntimeConfig()?.inference?.session;
391
392
  const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
392
393
  const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
393
- const directF16KVCacheWrite = resolveDirectF16KVCacheWrite({
394
- state,
395
- layerIdx,
396
- currentSeqLen,
397
- numTokens,
398
- numKVHeads,
399
- headDim,
400
- reusesSharedKV,
401
- storeSharedKV,
402
- diffusionGemmaDecoder,
403
- valueNorm: config.valueNorm === true,
404
- diagnosticsEnabled: qkNormRoPEDiagnostics,
405
- disableRoPE,
406
- });
394
+ const skipKVCacheWrites = state.skipKVCacheWrites === true;
395
+ const directF16KVCacheWrite = skipKVCacheWrites
396
+ ? null
397
+ : resolveDirectF16KVCacheWrite({
398
+ state,
399
+ layerIdx,
400
+ currentSeqLen,
401
+ numTokens,
402
+ numKVHeads,
403
+ headDim,
404
+ reusesSharedKV,
405
+ storeSharedKV,
406
+ diffusionGemmaDecoder,
407
+ valueNorm: config.valueNorm === true,
408
+ diagnosticsEnabled: qkNormRoPEDiagnostics,
409
+ disableRoPE,
410
+ });
407
411
  let kvCacheWriteFused = false;
408
412
  ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
409
413
  recorder,
@@ -649,7 +653,7 @@ export async function recordLayerAttentionGPU(
649
653
  }
650
654
 
651
655
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
652
- if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
656
+ if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
653
657
  if (kvCacheWriteFused) {
654
658
  state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
655
659
  } else if (state.kvCache.kvDtype === 'f16') {
@@ -741,7 +745,9 @@ export async function recordLayerAttentionGPU(
741
745
  }
742
746
  decoderKVState = kvState;
743
747
  } else {
744
- kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
748
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
749
+ forceCurrentTensors: skipKVCacheWrites,
750
+ });
745
751
  }
746
752
  const dispatchConfig = {
747
753
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
@@ -758,6 +764,7 @@ export async function recordLayerAttentionGPU(
758
764
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
759
765
  prefillFallbackNeedsCast, causalForAttention,
760
766
  } = dispatchParams;
767
+ const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
761
768
 
762
769
  // 5. Attention
763
770
 
@@ -767,6 +774,7 @@ export async function recordLayerAttentionGPU(
767
774
  usedFusedQKV, qTensor, kTensor, vTensor,
768
775
  ));
769
776
  const mergedSessionRec = getRuntimeConfig()?.inference?.session;
777
+ let attentionOutputGateFused = false;
770
778
 
771
779
  const attentionKernelRunners = {
772
780
  bdpa: async () => {
@@ -923,7 +931,18 @@ export async function recordLayerAttentionGPU(
923
931
  // path in ./run.js.
924
932
  const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
925
933
  const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
926
- return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
934
+ const useOutputGateFusion = canUseAttentionOutputGateFusion({
935
+ session: mergedSessionRec,
936
+ qGateTensor,
937
+ numTokens,
938
+ numHeads,
939
+ headDim,
940
+ cachedKDtype,
941
+ cachedVDtype,
942
+ kernelPath: attentionKernelPath,
943
+ diffusionGemmaDecoder,
944
+ });
945
+ const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
927
946
  seqLen: numTokens,
928
947
  kvLen: kvState.kvLenForAttention,
929
948
  numKVHeads,
@@ -939,10 +958,13 @@ export async function recordLayerAttentionGPU(
939
958
  kvLayout: kvState.kvLayout,
940
959
  kvPageTable: kvState.kvPageTable,
941
960
  kvPageSize: kvState.kvPageSize,
942
- kernelPath,
943
- useFlashPrefill: useFlashPrefillRec,
944
- useOrtFlashPrefill: useOrtFlashPrefillRec,
961
+ kernelPath: attentionKernelPath,
962
+ useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
963
+ useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
964
+ outputGate: useOutputGateFusion ? qGateTensor : null,
945
965
  });
966
+ attentionOutputGateFused = result?.outputGateFused === true;
967
+ return result;
946
968
  },
947
969
  };
948
970
  const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
@@ -980,13 +1002,14 @@ export async function recordLayerAttentionGPU(
980
1002
  }
981
1003
 
982
1004
  attnForProjection = attnOutput;
983
- if (qGateTensor) {
1005
+ if (qGateTensor && !attentionOutputGateFused) {
984
1006
  // Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
985
1007
  // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
986
1008
  const gateActivation = 'sigmoid';
987
1009
  attnForProjection = await recordSiLU(recorder, attnOutput, {
988
1010
  size: numTokens * numHeads * headDim,
989
1011
  gate: qGateTensor,
1012
+ useVec4: (numTokens * numHeads * headDim) % 4 === 0,
990
1013
  gateActivation,
991
1014
  inputActivation: 'identity',
992
1015
  swigluLimit: null,
@@ -66,6 +66,7 @@ import {
66
66
  isAttentionKvDtypeExplicit,
67
67
  } from './precision-contract.js';
68
68
  import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
69
+ import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
69
70
 
70
71
  const ATTENTION_DTYPE_LOGGED = new Set();
71
72
 
@@ -793,6 +794,8 @@ export async function runLayerAttentionGPU(
793
794
  cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
794
795
  prefillFallbackNeedsCast, causalForAttention,
795
796
  } = dispatchParams;
797
+ const mergedSession = getRuntimeConfig()?.inference?.session;
798
+ let attentionOutputGateFused = false;
796
799
 
797
800
  // 5. Attention (uses raw GPUBuffers)
798
801
 
@@ -991,9 +994,19 @@ export async function runLayerAttentionGPU(
991
994
  // session (state.runtimeConfig.inference.session is a stale snapshot —
992
995
  // see comment above at the rmsNorm fusion check). The kernel itself
993
996
  // enforces remaining preconditions (head_dim=256, etc.).
994
- const mergedSession = getRuntimeConfig()?.inference?.session;
995
997
  const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
996
998
  const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
999
+ const useOutputGateFusion = canUseAttentionOutputGateFusion({
1000
+ session: mergedSession,
1001
+ qGateTensor,
1002
+ numTokens,
1003
+ numHeads,
1004
+ headDim,
1005
+ cachedKDtype,
1006
+ cachedVDtype,
1007
+ kernelPath,
1008
+ diffusionGemmaDecoder,
1009
+ });
997
1010
  const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
998
1011
  seqLen: numTokens,
999
1012
  kvLen: kvState.kvLenForAttention,
@@ -1013,7 +1026,9 @@ export async function runLayerAttentionGPU(
1013
1026
  kernelPath,
1014
1027
  useFlashPrefill,
1015
1028
  useOrtFlashPrefill,
1029
+ outputGate: useOutputGateFusion ? qGateTensor : null,
1016
1030
  });
1031
+ attentionOutputGateFused = result?.outputGateFused === true;
1017
1032
  if (prefillFallbackNeedsCast) {
1018
1033
  if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
1019
1034
  if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
@@ -1068,7 +1083,7 @@ export async function runLayerAttentionGPU(
1068
1083
  }
1069
1084
 
1070
1085
  attnForProjection = attnOutput;
1071
- if (qGateTensor) {
1086
+ if (qGateTensor && !attentionOutputGateFused) {
1072
1087
  // The Qwen3_5 reference (HF transformers) full-attention path
1073
1088
  // applies sigmoid(gate) — NOT silu(gate) — to the attention
1074
1089
  // output regardless of how the architecture's gate label is
@@ -1089,6 +1104,7 @@ export async function runLayerAttentionGPU(
1089
1104
  attnForProjection = await runSiLU(attnOutput, {
1090
1105
  size: numTokens * numHeads * headDim,
1091
1106
  gate: qGateTensor,
1107
+ useVec4: (numTokens * numHeads * headDim) % 4 === 0,
1092
1108
  gateActivation,
1093
1109
  inputActivation: 'identity',
1094
1110
  swigluLimit: null,
@@ -1125,7 +1141,6 @@ export async function runLayerAttentionGPU(
1125
1141
  // Use fused o_proj + residual for decode when possible
1126
1142
  // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
1127
1143
  const oProjDtype = getWeightDtype(oProjBuf);
1128
- const mergedSession = getRuntimeConfig()?.inference?.session;
1129
1144
  const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
1130
1145
  allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
1131
1146
  hasResidual: Boolean(residualTensor),
@@ -114,14 +114,32 @@ export interface TensorInfo {
114
114
  export interface ParsedSessionSettings {
115
115
  prefillChunkSubmitMode: 'sync' | 'async' | null;
116
116
  prefillTokenChunkSize: number | null;
117
+ skipEmbeddingKVCacheWrites: boolean | null;
117
118
  useFlashPrefillAttention: boolean | null;
119
+ useLargeBatchF16F32FusedGateUp: boolean | null;
118
120
  useWideTileQ4KPrefill: boolean | null;
119
121
  useWideTileQ4KDecode: boolean | null;
120
122
  useSandwichRMSNormPairFusion: boolean | null;
121
123
  usePostFfnNextInputRMSNormPairFusion: boolean | null;
124
+ usePostAttnNormFusedGateUp: boolean | null;
125
+ useLinearAttentionABProjectionFusion: boolean | null;
126
+ useLinearAttentionQKVZProjectionFusion: boolean | null;
127
+ useLinearAttentionFusedDecodeCore: boolean | null;
128
+ useWideTileResidualFusion: boolean | null;
129
+ useFusedRmsnormWideTile: boolean | null;
122
130
  useFusedQKVSplitQKNorm: boolean | null;
123
131
  useFusedQKVSplitQKNormRoPE: boolean | null;
124
132
  retainQ4KMaterialization: boolean | null;
133
+ lmHeadArgmaxQ4K: {
134
+ useFullBlockFastPath?: boolean;
135
+ colsPerWorkgroup?: number;
136
+ threadsPerCol?: number;
137
+ } | null;
138
+ attentionDecodeOnline: {
139
+ workgroupSize?: 128 | 256;
140
+ useDirectContiguousKVLayout?: boolean;
141
+ useOutputGateFusion?: boolean;
142
+ } | null;
125
143
  }
126
144
 
127
145
  export type Manifest = RuntimeModelContract;
@@ -223,10 +223,22 @@ function resolveSessionSettings(inferenceConfig, modelId) {
223
223
  `"${String(tokenChunk)}"; expected null or a positive integer.`
224
224
  );
225
225
  }
226
+ const skipEmbeddingKVCacheWrites = session?.skipEmbeddingKVCacheWrites;
227
+ if (
228
+ skipEmbeddingKVCacheWrites !== undefined
229
+ && skipEmbeddingKVCacheWrites !== null
230
+ && typeof skipEmbeddingKVCacheWrites !== 'boolean'
231
+ ) {
232
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.skipEmbeddingKVCacheWrites "${String(skipEmbeddingKVCacheWrites)}"; expected boolean.`);
233
+ }
226
234
  const flash = session?.useFlashPrefillAttention;
227
235
  if (flash !== undefined && flash !== null && typeof flash !== 'boolean') {
228
236
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useFlashPrefillAttention "${String(flash)}"; expected boolean.`);
229
237
  }
238
+ const largeBatchF16F32FusedGateUp = session?.useLargeBatchF16F32FusedGateUp;
239
+ if (largeBatchF16F32FusedGateUp !== undefined && largeBatchF16F32FusedGateUp !== null && typeof largeBatchF16F32FusedGateUp !== 'boolean') {
240
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLargeBatchF16F32FusedGateUp "${String(largeBatchF16F32FusedGateUp)}"; expected boolean.`);
241
+ }
230
242
  const wide = session?.useWideTileQ4KPrefill;
231
243
  if (wide !== undefined && wide !== null && typeof wide !== 'boolean') {
232
244
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileQ4KPrefill "${String(wide)}"; expected boolean.`);
@@ -243,6 +255,30 @@ function resolveSessionSettings(inferenceConfig, modelId) {
243
255
  if (postFfnNextInputRmsNormPair !== undefined && postFfnNextInputRmsNormPair !== null && typeof postFfnNextInputRmsNormPair !== 'boolean') {
244
256
  throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostFfnNextInputRMSNormPairFusion "${String(postFfnNextInputRmsNormPair)}"; expected boolean.`);
245
257
  }
258
+ const postAttnNormFusedGateUp = session?.usePostAttnNormFusedGateUp;
259
+ if (postAttnNormFusedGateUp !== undefined && postAttnNormFusedGateUp !== null && typeof postAttnNormFusedGateUp !== 'boolean') {
260
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.usePostAttnNormFusedGateUp "${String(postAttnNormFusedGateUp)}"; expected boolean.`);
261
+ }
262
+ const linearAttentionABProjectionFusion = session?.useLinearAttentionABProjectionFusion;
263
+ if (linearAttentionABProjectionFusion !== undefined && linearAttentionABProjectionFusion !== null && typeof linearAttentionABProjectionFusion !== 'boolean') {
264
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionABProjectionFusion "${String(linearAttentionABProjectionFusion)}"; expected boolean.`);
265
+ }
266
+ const linearAttentionQKVZProjectionFusion = session?.useLinearAttentionQKVZProjectionFusion;
267
+ if (linearAttentionQKVZProjectionFusion !== undefined && linearAttentionQKVZProjectionFusion !== null && typeof linearAttentionQKVZProjectionFusion !== 'boolean') {
268
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionQKVZProjectionFusion "${String(linearAttentionQKVZProjectionFusion)}"; expected boolean.`);
269
+ }
270
+ const linearAttentionFusedDecodeCore = session?.useLinearAttentionFusedDecodeCore;
271
+ if (linearAttentionFusedDecodeCore !== undefined && linearAttentionFusedDecodeCore !== null && typeof linearAttentionFusedDecodeCore !== 'boolean') {
272
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useLinearAttentionFusedDecodeCore "${String(linearAttentionFusedDecodeCore)}"; expected boolean.`);
273
+ }
274
+ const wideTileResidualFusion = session?.useWideTileResidualFusion;
275
+ if (wideTileResidualFusion !== undefined && wideTileResidualFusion !== null && typeof wideTileResidualFusion !== 'boolean') {
276
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useWideTileResidualFusion "${String(wideTileResidualFusion)}"; expected boolean.`);
277
+ }
278
+ const fusedRmsnormWideTile = session?.useFusedRmsnormWideTile;
279
+ if (fusedRmsnormWideTile !== undefined && fusedRmsnormWideTile !== null && typeof fusedRmsnormWideTile !== 'boolean') {
280
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedRmsnormWideTile "${String(fusedRmsnormWideTile)}"; expected boolean.`);
281
+ }
246
282
  const fusedQKVSplitQKNorm = session?.useFusedQKVSplitQKNorm;
247
283
  if (fusedQKVSplitQKNorm !== undefined && fusedQKVSplitQKNorm !== null && typeof fusedQKVSplitQKNorm !== 'boolean') {
248
284
  throw new Error(`Manifest "${modelId}" has invalid inference.session.useFusedQKVSplitQKNorm "${String(fusedQKVSplitQKNorm)}"; expected boolean.`);
@@ -255,17 +291,81 @@ function resolveSessionSettings(inferenceConfig, modelId) {
255
291
  if (retain !== undefined && retain !== null && typeof retain !== 'boolean') {
256
292
  throw new Error(`Manifest "${modelId}" has invalid inference.session.retainQ4KMaterialization "${String(retain)}"; expected boolean.`);
257
293
  }
294
+ const lmHeadArgmaxQ4K = session?.lmHeadArgmaxQ4K;
295
+ if (lmHeadArgmaxQ4K !== undefined && lmHeadArgmaxQ4K !== null) {
296
+ if (typeof lmHeadArgmaxQ4K !== 'object' || Array.isArray(lmHeadArgmaxQ4K)) {
297
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K; expected object or null.`);
298
+ }
299
+ const fullBlockFastPath = lmHeadArgmaxQ4K.useFullBlockFastPath;
300
+ if (fullBlockFastPath !== undefined && typeof fullBlockFastPath !== 'boolean') {
301
+ throw new Error(
302
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.useFullBlockFastPath ` +
303
+ `"${String(fullBlockFastPath)}"; expected boolean.`
304
+ );
305
+ }
306
+ const colsPerWorkgroup = lmHeadArgmaxQ4K.colsPerWorkgroup;
307
+ if (colsPerWorkgroup !== undefined && (!Number.isInteger(colsPerWorkgroup) || colsPerWorkgroup <= 0)) {
308
+ throw new Error(
309
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.colsPerWorkgroup ` +
310
+ `"${String(colsPerWorkgroup)}"; expected positive integer.`
311
+ );
312
+ }
313
+ const threadsPerCol = lmHeadArgmaxQ4K.threadsPerCol;
314
+ if (threadsPerCol !== undefined && (!Number.isInteger(threadsPerCol) || threadsPerCol <= 0)) {
315
+ throw new Error(
316
+ `Manifest "${modelId}" has invalid inference.session.lmHeadArgmaxQ4K.threadsPerCol ` +
317
+ `"${String(threadsPerCol)}"; expected positive integer.`
318
+ );
319
+ }
320
+ }
321
+ const attentionDecodeOnline = session?.attentionDecodeOnline;
322
+ if (attentionDecodeOnline !== undefined && attentionDecodeOnline !== null) {
323
+ if (typeof attentionDecodeOnline !== 'object' || Array.isArray(attentionDecodeOnline)) {
324
+ throw new Error(`Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline; expected object or null.`);
325
+ }
326
+ const workgroupSize = attentionDecodeOnline.workgroupSize;
327
+ if (workgroupSize !== undefined && workgroupSize !== 128 && workgroupSize !== 256) {
328
+ throw new Error(
329
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.workgroupSize ` +
330
+ `"${String(workgroupSize)}"; expected 128 or 256.`
331
+ );
332
+ }
333
+ const directKv = attentionDecodeOnline.useDirectContiguousKVLayout;
334
+ if (directKv !== undefined && typeof directKv !== 'boolean') {
335
+ throw new Error(
336
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useDirectContiguousKVLayout ` +
337
+ `"${String(directKv)}"; expected boolean.`
338
+ );
339
+ }
340
+ const outputGateFusion = attentionDecodeOnline.useOutputGateFusion;
341
+ if (outputGateFusion !== undefined && typeof outputGateFusion !== 'boolean') {
342
+ throw new Error(
343
+ `Manifest "${modelId}" has invalid inference.session.attentionDecodeOnline.useOutputGateFusion ` +
344
+ `"${String(outputGateFusion)}"; expected boolean.`
345
+ );
346
+ }
347
+ }
258
348
  return {
259
349
  prefillChunkSubmitMode: submit ?? null,
260
350
  prefillTokenChunkSize: tokenChunk ?? null,
351
+ skipEmbeddingKVCacheWrites: skipEmbeddingKVCacheWrites ?? null,
261
352
  useFlashPrefillAttention: flash ?? null,
353
+ useLargeBatchF16F32FusedGateUp: largeBatchF16F32FusedGateUp ?? null,
262
354
  useWideTileQ4KPrefill: wide ?? null,
263
355
  useWideTileQ4KDecode: wideDecode ?? null,
264
356
  useSandwichRMSNormPairFusion: sandwichRmsNormPair ?? null,
265
357
  usePostFfnNextInputRMSNormPairFusion: postFfnNextInputRmsNormPair ?? null,
358
+ usePostAttnNormFusedGateUp: postAttnNormFusedGateUp ?? null,
359
+ useLinearAttentionABProjectionFusion: linearAttentionABProjectionFusion ?? null,
360
+ useLinearAttentionQKVZProjectionFusion: linearAttentionQKVZProjectionFusion ?? null,
361
+ useLinearAttentionFusedDecodeCore: linearAttentionFusedDecodeCore ?? null,
362
+ useWideTileResidualFusion: wideTileResidualFusion ?? null,
363
+ useFusedRmsnormWideTile: fusedRmsnormWideTile ?? null,
266
364
  useFusedQKVSplitQKNorm: fusedQKVSplitQKNorm ?? null,
267
365
  useFusedQKVSplitQKNormRoPE: fusedQKVSplitQKNormRoPE ?? null,
268
366
  retainQ4KMaterialization: retain ?? null,
367
+ lmHeadArgmaxQ4K: lmHeadArgmaxQ4K ?? null,
368
+ attentionDecodeOnline: attentionDecodeOnline ?? null,
269
369
  };
270
370
  }
271
371
 
@@ -63,7 +63,7 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
63
63
  }
64
64
  }
65
65
 
66
- function resolveEmbeddingScale(config, hiddenSize) {
66
+ export function resolveEmbeddingScale(config, hiddenSize) {
67
67
  const embeddingScale = config.embeddingScale;
68
68
  const scaleEmbeddings = config.scaleEmbeddings;
69
69
  if (embeddingScale === undefined) {
@@ -21,6 +21,7 @@ export interface StaticExecutionPlan {
21
21
  defaultMaxTokens: number;
22
22
  readbackInterval: number | null;
23
23
  readbackMode: 'sequential' | 'overlapped' | 'auto';
24
+ maxBatchDecodeTokens?: number | null;
24
25
  ringTokens: number | null;
25
26
  ringStop: number | null;
26
27
  ringStaging: number | null;
@@ -53,6 +54,7 @@ export interface ExecutionSessionPlan {
53
54
  maxTokens: number;
54
55
  readbackInterval: number | null;
55
56
  readbackMode: 'sequential' | 'overlapped' | 'auto';
57
+ maxBatchDecodeTokens?: number | null;
56
58
  ringTokens: number | null;
57
59
  ringStop: number | null;
58
60
  ringStaging: number | null;