doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -0,0 +1,208 @@
1
+ import { getRuntimeConfig } from '../../../config/runtime.js';
2
+ import { getDevice } from '../../../gpu/device.js';
3
+ import { createWeightBuffer, isWeightBuffer } from '../../../gpu/weight-buffer.js';
4
+ import { Q4K_BLOCK_BYTES, QK_K } from '../../../config/schema/index.js';
5
+
6
+ function shapeMatches(weight, rows, cols) {
7
+ return Array.isArray(weight?.shape)
8
+ && weight.shape.length === 2
9
+ && Number(weight.shape[0]) === rows
10
+ && Number(weight.shape[1]) === cols;
11
+ }
12
+
13
+ function canUseLinearAttentionABProjectionFusion({
14
+ phase,
15
+ numTokens,
16
+ debugProbes,
17
+ operatorDiagnostics,
18
+ }) {
19
+ const session = getRuntimeConfig()?.inference?.session;
20
+ if (session?.useLinearAttentionABProjectionFusion !== true) {
21
+ return false;
22
+ }
23
+ if (phase !== 'decode' || numTokens !== 1) {
24
+ return false;
25
+ }
26
+ if (debugProbes?.length || operatorDiagnostics?.enabled === true) {
27
+ return false;
28
+ }
29
+ return true;
30
+ }
31
+
32
+ export function resolveLinearAttentionABProjection(layerWeights, options) {
33
+ const {
34
+ phase,
35
+ numTokens,
36
+ hiddenSize,
37
+ numVHeads,
38
+ layerIdx,
39
+ debugProbes,
40
+ operatorDiagnostics,
41
+ } = options;
42
+ if (!canUseLinearAttentionABProjectionFusion({
43
+ phase,
44
+ numTokens,
45
+ debugProbes,
46
+ operatorDiagnostics,
47
+ })) {
48
+ return null;
49
+ }
50
+ if (layerWeights.linearABProj && shapeMatches(layerWeights.linearABProj, numVHeads * 2, hiddenSize)) {
51
+ return {
52
+ weight: layerWeights.linearABProj,
53
+ outDim: numVHeads * 2,
54
+ bProjOffsetElements: numTokens * numVHeads,
55
+ };
56
+ }
57
+
58
+ const aWeight = layerWeights.linearInProjA;
59
+ const bWeight = layerWeights.linearInProjB;
60
+ if (
61
+ !isWeightBuffer(aWeight)
62
+ || !isWeightBuffer(bWeight)
63
+ || aWeight.dtype !== 'f16'
64
+ || bWeight.dtype !== 'f16'
65
+ || aWeight.layout !== 'row'
66
+ || bWeight.layout !== 'row'
67
+ || !shapeMatches(aWeight, numVHeads, hiddenSize)
68
+ || !shapeMatches(bWeight, numVHeads, hiddenSize)
69
+ ) {
70
+ return null;
71
+ }
72
+
73
+ const bytesPerWeight = numVHeads * hiddenSize * 2;
74
+ if (aWeight.buffer.size < bytesPerWeight || bWeight.buffer.size < bytesPerWeight) {
75
+ return null;
76
+ }
77
+ const device = getDevice();
78
+ if (!device) {
79
+ return null;
80
+ }
81
+
82
+ const buffer = device.createBuffer({
83
+ label: `L${layerIdx}.linear_ab_proj_weight`,
84
+ size: bytesPerWeight * 2,
85
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
86
+ });
87
+ const encoder = device.createCommandEncoder({ label: `L${layerIdx}.linear_ab_proj_weight_pack` });
88
+ encoder.copyBufferToBuffer(aWeight.buffer, 0, buffer, 0, bytesPerWeight);
89
+ encoder.copyBufferToBuffer(bWeight.buffer, 0, buffer, bytesPerWeight, bytesPerWeight);
90
+ device.queue.submit([encoder.finish()]);
91
+
92
+ layerWeights.linearABProj = createWeightBuffer(
93
+ buffer,
94
+ 'f16',
95
+ 'row',
96
+ [numVHeads * 2, hiddenSize],
97
+ `L${layerIdx}.linear_ab_proj_weight`
98
+ );
99
+ return {
100
+ weight: layerWeights.linearABProj,
101
+ outDim: numVHeads * 2,
102
+ bProjOffsetElements: numTokens * numVHeads,
103
+ };
104
+ }
105
+
106
+ function canUseLinearAttentionQKVZProjectionFusion({
107
+ phase,
108
+ numTokens,
109
+ debugProbes,
110
+ operatorDiagnostics,
111
+ }) {
112
+ const session = getRuntimeConfig()?.inference?.session;
113
+ if (session?.useLinearAttentionQKVZProjectionFusion !== true) {
114
+ return false;
115
+ }
116
+ if (phase !== 'decode' || numTokens !== 1) {
117
+ return false;
118
+ }
119
+ if (debugProbes?.length || operatorDiagnostics?.enabled === true) {
120
+ return false;
121
+ }
122
+ return true;
123
+ }
124
+
125
+ function q4kRowBytes(hiddenSize) {
126
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0) {
127
+ return null;
128
+ }
129
+ return Math.ceil(hiddenSize / QK_K) * Q4K_BLOCK_BYTES;
130
+ }
131
+
132
+ export function resolveLinearAttentionQKVZProjection(layerWeights, options) {
133
+ const {
134
+ phase,
135
+ numTokens,
136
+ hiddenSize,
137
+ convDim,
138
+ valueDim,
139
+ layerIdx,
140
+ debugProbes,
141
+ operatorDiagnostics,
142
+ } = options;
143
+ if (!canUseLinearAttentionQKVZProjectionFusion({
144
+ phase,
145
+ numTokens,
146
+ debugProbes,
147
+ operatorDiagnostics,
148
+ })) {
149
+ return null;
150
+ }
151
+ if (layerWeights.linearQKVZProj && shapeMatches(layerWeights.linearQKVZProj, convDim + valueDim, hiddenSize)) {
152
+ return {
153
+ weight: layerWeights.linearQKVZProj,
154
+ outDim: convDim + valueDim,
155
+ };
156
+ }
157
+
158
+ const qkvWeight = layerWeights.qkvProj;
159
+ const zWeight = layerWeights.linearInProjZ;
160
+ if (
161
+ !isWeightBuffer(qkvWeight)
162
+ || !isWeightBuffer(zWeight)
163
+ || qkvWeight.dtype !== 'q4k'
164
+ || zWeight.dtype !== 'q4k'
165
+ || qkvWeight.layout !== 'row'
166
+ || zWeight.layout !== 'row'
167
+ || !shapeMatches(qkvWeight, convDim, hiddenSize)
168
+ || !shapeMatches(zWeight, valueDim, hiddenSize)
169
+ ) {
170
+ return null;
171
+ }
172
+
173
+ const rowBytes = q4kRowBytes(hiddenSize);
174
+ if (rowBytes == null) {
175
+ return null;
176
+ }
177
+ const qkvBytes = convDim * rowBytes;
178
+ const zBytes = valueDim * rowBytes;
179
+ if (qkvWeight.buffer.size < qkvBytes || zWeight.buffer.size < zBytes) {
180
+ return null;
181
+ }
182
+ const device = getDevice();
183
+ if (!device) {
184
+ return null;
185
+ }
186
+
187
+ const buffer = device.createBuffer({
188
+ label: `L${layerIdx}.linear_qkvz_proj_weight`,
189
+ size: qkvBytes + zBytes,
190
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
191
+ });
192
+ const encoder = device.createCommandEncoder({ label: `L${layerIdx}.linear_qkvz_proj_weight_pack` });
193
+ encoder.copyBufferToBuffer(qkvWeight.buffer, 0, buffer, 0, qkvBytes);
194
+ encoder.copyBufferToBuffer(zWeight.buffer, 0, buffer, qkvBytes, zBytes);
195
+ device.queue.submit([encoder.finish()]);
196
+
197
+ layerWeights.linearQKVZProj = createWeightBuffer(
198
+ buffer,
199
+ 'q4k',
200
+ 'row',
201
+ [convDim + valueDim, hiddenSize],
202
+ `L${layerIdx}.linear_qkvz_proj_weight`
203
+ );
204
+ return {
205
+ weight: layerWeights.linearQKVZProj,
206
+ outDim: convDim + valueDim,
207
+ };
208
+ }
@@ -73,6 +73,7 @@ export interface RunLinearAttentionLayerOptions {
73
73
  weight: GPUBuffer | Float32Array | ArrayBuffer,
74
74
  label: string
75
75
  ) => GPUBuffer;
76
+ precomputedInputNorm?: Tensor | null;
76
77
  debugProbes?: ProbeConfigSchema[] | null;
77
78
  recorder?: CommandRecorder | null;
78
79
  }
@@ -1,9 +1,22 @@
1
1
  import { getBufferDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
2
- import { recordMatmul, recordRMSNorm, runMatmul, runRMSNorm, castF32ToF16, recordCastF32ToF16 } from '../../../gpu/kernel-selector.js';
2
+ import {
3
+ recordMatmul,
4
+ recordRMSNorm,
5
+ runMatmul,
6
+ runRMSNorm,
7
+ castF32ToF16,
8
+ castF16ToF32,
9
+ recordCastF32ToF16,
10
+ recordCastF16ToF32,
11
+ } from '../../../gpu/kernel-selector.js';
3
12
  import { readBuffer, releaseBuffer, uploadData, acquireBuffer } from '../../../memory/buffer-pool.js';
4
13
  import { log } from '../../../debug/index.js';
5
14
  import { decodeReadback, f16ToF32 } from './debug-utils/index.js';
6
15
  import { runLinearAttentionCoreGPU } from '../../../gpu/kernels/linear-attention-core.js';
16
+ import {
17
+ resolveLinearAttentionABProjection,
18
+ resolveLinearAttentionQKVZProjection,
19
+ } from './linear-attention-ab-fusion.js';
7
20
  import { runProbes } from './probes.js';
8
21
  import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
9
22
  import { dequantizeQ4KM } from '../../../converter/quantizer.js';
@@ -656,9 +669,32 @@ async function projectLinearTensor({
656
669
  ? await recordCastF32ToF16(recorder, inputTensor)
657
670
  : await castF32ToF16(inputTensor);
658
671
  }
672
+ const settleProjectionOutputDtype = async (result) => {
673
+ if (result.dtype === resolvedOutputDtype) {
674
+ return result;
675
+ }
676
+ assertImplicitDtypeTransitionAllowed({
677
+ executionPolicies,
678
+ fromDtype: result.dtype,
679
+ toDtype: resolvedOutputDtype,
680
+ op: role,
681
+ detail: 'Linear attention projection returned a kernel-selected dtype different from the declared projection output dtype.',
682
+ });
683
+ if (resolvedOutputDtype === 'f16') {
684
+ const casted = recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result);
685
+ releaseOrTrackBuffer(recorder, result.buffer);
686
+ return casted;
687
+ }
688
+ if (resolvedOutputDtype === 'f32') {
689
+ const casted = recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result);
690
+ releaseOrTrackBuffer(recorder, result.buffer);
691
+ return casted;
692
+ }
693
+ throw new Error(`Unsupported linear_attention projection output dtype "${String(resolvedOutputDtype)}".`);
694
+ };
659
695
  try {
660
696
  if (recorder) {
661
- return await recordMatmul(recorder, matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
697
+ const result = await recordMatmul(recorder, matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
662
698
  transposeB: 'auto',
663
699
  role,
664
700
  layerIdx,
@@ -666,8 +702,9 @@ async function projectLinearTensor({
666
702
  outputDtype: resolvedOutputDtype,
667
703
  executionPolicies,
668
704
  });
705
+ return await settleProjectionOutputDtype(result);
669
706
  }
670
- return await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
707
+ const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
671
708
  transposeB: 'auto',
672
709
  role,
673
710
  layerIdx,
@@ -675,6 +712,7 @@ async function projectLinearTensor({
675
712
  outputDtype: resolvedOutputDtype,
676
713
  executionPolicies,
677
714
  });
715
+ return await settleProjectionOutputDtype(result);
678
716
  } finally {
679
717
  if (matmulInput !== inputTensor) {
680
718
  releaseOrTrackBuffer(recorder, matmulInput.buffer);
@@ -683,6 +721,31 @@ async function projectLinearTensor({
683
721
  }
684
722
  }
685
723
 
724
+ async function settleLinearAttentionCoreInputDtype(tensor, targetDtype, options) {
725
+ if (tensor.dtype === targetDtype) {
726
+ return tensor;
727
+ }
728
+ const {
729
+ recorder,
730
+ executionPolicies,
731
+ role,
732
+ } = options;
733
+ assertImplicitDtypeTransitionAllowed({
734
+ executionPolicies,
735
+ fromDtype: tensor.dtype,
736
+ toDtype: targetDtype,
737
+ op: role,
738
+ detail: 'Linear attention core requires qkv/z/a/b tensors to share one input dtype.',
739
+ });
740
+ if (targetDtype === 'f16') {
741
+ return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
742
+ }
743
+ if (targetDtype === 'f32') {
744
+ return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
745
+ }
746
+ throw new Error(`Unsupported linear_attention core input dtype "${String(targetDtype)}".`);
747
+ }
748
+
686
749
  export function hasLinearAttentionLayers(layerTypes) {
687
750
  if (!Array.isArray(layerTypes)) return false;
688
751
  for (let i = 0; i < layerTypes.length; i++) {
@@ -771,6 +834,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
771
834
  getNormWeightBuffer,
772
835
  recorder,
773
836
  executionPolicies = null,
837
+ precomputedInputNorm = null,
774
838
  } = options;
775
839
 
776
840
  if (!layerWeights) {
@@ -808,7 +872,20 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
808
872
  let normedTensor = inputTensor;
809
873
  let normedCreated = false;
810
874
 
811
- if (layerWeights.inputNorm) {
875
+ if (precomputedInputNorm) {
876
+ if (
877
+ phase !== 'decode'
878
+ || numTokens !== 1
879
+ || !layerWeights.inputNorm
880
+ || precomputedInputNorm.dtype !== inputTensor.dtype
881
+ || precomputedInputNorm.dtype !== 'f32'
882
+ ) {
883
+ releaseOrTrackBuffer(recorder, precomputedInputNorm.buffer);
884
+ throw new Error(`linear_attention layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
885
+ }
886
+ normedTensor = precomputedInputNorm;
887
+ normedCreated = true;
888
+ } else if (layerWeights.inputNorm) {
812
889
  const normWeightBuffer = getNormWeightBuffer(layerWeights.inputNorm, `L${layerIdx}.linear_input_norm`);
813
890
  try {
814
891
  if (recorder) {
@@ -832,7 +909,35 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
832
909
  }
833
910
  }
834
911
 
835
- const qkvTensor = await projectLinearTensor({
912
+ const qkvzProjection = resolveLinearAttentionQKVZProjection(layerWeights, {
913
+ phase,
914
+ numTokens,
915
+ hiddenSize,
916
+ numVHeads: projectionLayout.numVHeads,
917
+ convDim: projectionLayout.convDim,
918
+ valueDim: projectionLayout.valueDim,
919
+ layerIdx,
920
+ debugProbes: options.debugProbes,
921
+ operatorDiagnostics: options.operatorDiagnostics,
922
+ });
923
+ const qkvzTensor = qkvzProjection
924
+ ? await projectLinearTensor({
925
+ inputTensor: normedTensor,
926
+ sourceWeight: qkvzProjection.weight,
927
+ role: 'linear_qkvz_proj',
928
+ phase,
929
+ outDim: qkvzProjection.outDim,
930
+ numTokens,
931
+ hiddenSize,
932
+ layerIdx,
933
+ kernelPath,
934
+ outputDtype: projectionDtype,
935
+ getWeightBuffer,
936
+ recorder,
937
+ executionPolicies,
938
+ })
939
+ : null;
940
+ const qkvTensor = qkvzTensor ?? await projectLinearTensor({
836
941
  inputTensor: normedTensor,
837
942
  sourceWeight: layerWeights.qkvProj,
838
943
  role: 'linear_qkv_proj',
@@ -847,7 +952,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
847
952
  recorder,
848
953
  executionPolicies,
849
954
  });
850
- const zTensor = await projectLinearTensor({
955
+ const zTensor = qkvzTensor ?? await projectLinearTensor({
851
956
  inputTensor: normedTensor,
852
957
  sourceWeight: layerWeights.linearInProjZ,
853
958
  role: 'linear_z_proj',
@@ -862,7 +967,33 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
862
967
  recorder,
863
968
  executionPolicies,
864
969
  });
865
- const aTensor = await projectLinearTensor({
970
+ const abProjection = resolveLinearAttentionABProjection(layerWeights, {
971
+ phase,
972
+ numTokens,
973
+ hiddenSize,
974
+ numVHeads: projectionLayout.numVHeads,
975
+ layerIdx,
976
+ debugProbes: options.debugProbes,
977
+ operatorDiagnostics: options.operatorDiagnostics,
978
+ });
979
+ const abTensor = abProjection
980
+ ? await projectLinearTensor({
981
+ inputTensor: normedTensor,
982
+ sourceWeight: abProjection.weight,
983
+ role: 'linear_ab_proj',
984
+ phase,
985
+ outDim: abProjection.outDim,
986
+ numTokens,
987
+ hiddenSize,
988
+ layerIdx,
989
+ kernelPath,
990
+ outputDtype: projectionDtype,
991
+ getWeightBuffer,
992
+ recorder,
993
+ executionPolicies,
994
+ })
995
+ : null;
996
+ const aTensor = abTensor ?? await projectLinearTensor({
866
997
  inputTensor: normedTensor,
867
998
  sourceWeight: layerWeights.linearInProjA,
868
999
  role: 'linear_a_proj',
@@ -877,7 +1008,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
877
1008
  recorder,
878
1009
  executionPolicies,
879
1010
  });
880
- const bTensor = await projectLinearTensor({
1011
+ const bTensor = abTensor ?? await projectLinearTensor({
881
1012
  inputTensor: normedTensor,
882
1013
  sourceWeight: layerWeights.linearInProjB,
883
1014
  role: 'linear_b_proj',
@@ -909,6 +1040,10 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
909
1040
  layerOutputDtype,
910
1041
  'outputDtype'
911
1042
  );
1043
+ let coreQkvTensor = qkvTensor;
1044
+ let coreZTensor = zTensor;
1045
+ let coreATensor = aTensor;
1046
+ let coreBTensor = bTensor;
912
1047
 
913
1048
  try {
914
1049
  await runProbes('linear_qkv_proj', qkvTensor.buffer, {
@@ -947,11 +1082,27 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
947
1082
  operatorDiagnostics: options.operatorDiagnostics,
948
1083
  dtype: bTensor.dtype,
949
1084
  });
1085
+ const coreInputDtype = qkvTensor.dtype;
1086
+ coreZTensor = await settleLinearAttentionCoreInputDtype(zTensor, coreInputDtype, {
1087
+ recorder,
1088
+ executionPolicies,
1089
+ role: 'linear_z_proj',
1090
+ });
1091
+ coreATensor = await settleLinearAttentionCoreInputDtype(aTensor, coreInputDtype, {
1092
+ recorder,
1093
+ executionPolicies,
1094
+ role: 'linear_a_proj',
1095
+ });
1096
+ coreBTensor = await settleLinearAttentionCoreInputDtype(bTensor, coreInputDtype, {
1097
+ recorder,
1098
+ executionPolicies,
1099
+ role: 'linear_b_proj',
1100
+ });
950
1101
  const coreTensor = await runLinearAttentionCoreGPU(
951
- qkvTensor,
952
- zTensor,
953
- aTensor,
954
- bTensor,
1102
+ coreQkvTensor,
1103
+ coreZTensor,
1104
+ coreATensor,
1105
+ coreBTensor,
955
1106
  layerState,
956
1107
  {
957
1108
  numTokens,
@@ -960,6 +1111,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
960
1111
  qkL2NormEps: QK_L2NORM_EPS,
961
1112
  recorder,
962
1113
  executionPolicies,
1114
+ abPacked: abProjection != null,
1115
+ qkvzPacked: qkvzProjection != null,
1116
+ bProjOffsetElements: abProjection?.bProjOffsetElements ?? 0,
963
1117
  }
964
1118
  );
965
1119
  await runProbes('linear_core_out', coreTensor.buffer, {
@@ -1017,9 +1171,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
1017
1171
  if (normedCreated) {
1018
1172
  releaseOrTrackBuffer(recorder, normedTensor.buffer);
1019
1173
  }
1020
- releaseOrTrackBuffer(recorder, qkvTensor.buffer);
1021
- releaseOrTrackBuffer(recorder, zTensor.buffer);
1022
- releaseOrTrackBuffer(recorder, aTensor.buffer);
1023
- releaseOrTrackBuffer(recorder, bTensor.buffer);
1174
+ const released = new Set();
1175
+ for (const tensor of [qkvTensor, zTensor, aTensor, bTensor, coreQkvTensor, coreZTensor, coreATensor, coreBTensor]) {
1176
+ const buffer = tensor?.buffer ?? null;
1177
+ if (!buffer || released.has(buffer)) continue;
1178
+ released.add(buffer);
1179
+ releaseOrTrackBuffer(recorder, buffer);
1180
+ }
1024
1181
  }
1025
1182
  }
@@ -5,7 +5,7 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
5
5
  import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
6
6
  import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
7
  import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
- import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
8
+ import { recordLmHeadArgmax } from '../../../../gpu/kernels/lm-head-argmax.js';
9
9
  import { createTensor } from '../../../../gpu/tensor.js';
10
10
  import {
11
11
  castF16ToF32,
@@ -1176,7 +1176,7 @@ export async function recordGreedyLmHeadArgmaxGPU(
1176
1176
  config,
1177
1177
  operatorDiagnostics
1178
1178
  );
1179
- const outputBuffer = await recordLmHeadArgmaxF16(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1179
+ const outputBuffer = await recordLmHeadArgmax(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1180
1180
  vocabSize: tail.matmulVocabSize,
1181
1181
  hiddenSize: tail.hiddenSize,
1182
1182
  padTokenId: options.padTokenId,
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -50,6 +50,28 @@ function resolveLmHeadMatmulRole(phase) {
50
50
  return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
51
51
  }
52
52
 
53
+ function normalizeSelectedLogitTokenIds(value, vocabSize) {
54
+ if (value == null) {
55
+ return null;
56
+ }
57
+ if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
58
+ throw new Error('[Logits] selectedTokenIds must be an array or typed array.');
59
+ }
60
+ const tokenIds = Array.from(value, (entry, index) => {
61
+ const tokenId = Number(entry);
62
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
63
+ throw new Error(
64
+ `[Logits] selectedTokenIds[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`
65
+ );
66
+ }
67
+ return tokenId;
68
+ });
69
+ if (tokenIds.length === 0) {
70
+ throw new Error('[Logits] selectedTokenIds must not be empty.');
71
+ }
72
+ return tokenIds;
73
+ }
74
+
53
75
  async function coerceTensorDtype(tensor, targetDtype, options = {}) {
54
76
  if (!targetDtype || tensor.dtype === targetDtype) {
55
77
  return tensor;
@@ -196,6 +218,7 @@ export async function computeLogits(
196
218
  matmulVocabSize = dims.vocabSize;
197
219
  }
198
220
  }
221
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(options?.selectedTokenIds, matmulVocabSize);
199
222
 
200
223
  // Check if input is GPU buffer
201
224
  const inputIsGPU = isGpuBufferInstance(hiddenStates);
@@ -204,6 +227,12 @@ export async function computeLogits(
204
227
  if (isTraceEnabled('logits')) {
205
228
  trace.logits(`LOGITS_PATH: device=${!!device}, useGPU=${useGPU}, taking ${(!device || !useGPU) ? 'CPU' : 'GPU'} path`);
206
229
  }
230
+ if (selectedTokenIds && (!device || !useGPU)) {
231
+ throw new Error('[Logits] selectedTokenIds requires the GPU logits path.');
232
+ }
233
+ if (selectedTokenIds && (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
234
+ throw new Error('[Logits] selectedTokenIds requires a single GPU-resident LM head weight buffer.');
235
+ }
207
236
  if (!device || !useGPU) {
208
237
 
209
238
  let cpuHiddenStates;
@@ -481,6 +510,38 @@ export async function computeLogits(
481
510
  matmulInputOwned = true;
482
511
  }
483
512
 
513
+ if (selectedTokenIds) {
514
+ if (lastPositionOnly !== true) {
515
+ throw new Error('[Logits] selectedTokenIds requires lastPositionOnly=true.');
516
+ }
517
+ const selected = await runLmHeadSelectLogitsF16(matmulInputTensor, lmHeadBuffer, {
518
+ device,
519
+ hiddenSize,
520
+ vocabSize: matmulVocabSize,
521
+ tokenIds: selectedTokenIds,
522
+ hiddenOffset: 0,
523
+ logitSoftcap: config.finalLogitSoftcapping == null ? 0 : Number(config.finalLogitSoftcapping),
524
+ });
525
+ const selectedBytes = selectedTokenIds.length * Float32Array.BYTES_PER_ELEMENT;
526
+ const selectedData = await readBufferWithCleanup(selected.outputBuffer, selectedBytes, () => {
527
+ releaseBuffer(selected.outputBuffer);
528
+ releaseBuffer(selected.tokenIdBuffer);
529
+ if (inputBufferOwned) releaseBuffer(inputBuffer);
530
+ releaseBuffer(finalNormTensor.buffer);
531
+ if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
532
+ if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
533
+ if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
534
+ });
535
+ const selectedLogits = new Float32Array(selectedData);
536
+ await runProbes('logits_final', selectedLogits, {
537
+ numTokens: 1,
538
+ hiddenSize: selectedTokenIds.length,
539
+ probes: debugProbes,
540
+ operatorDiagnostics,
541
+ });
542
+ return selectedLogits;
543
+ }
544
+
484
545
  // HuggingFace models store lm_head as [vocabSize, hiddenSize], so transposeB=true
485
546
  const logitsTensor = await runMatmul(matmulInputTensor, lmHeadBuffer, matmulRows, matmulVocabSize, hiddenSize, {
486
547
  transposeB: 'auto',
@@ -155,6 +155,11 @@ function buildResolvedDecodeLoopRuntimePatch(
155
155
  );
156
156
  }
157
157
  const submitLatencyThresholdMs = decodeLoop.submitLatencyThresholdMs ?? null;
158
+ const maxBatchDecodeTokens = parseSessionDecodeLoopOptionalPositiveInt(
159
+ decodeLoop.maxBatchDecodeTokens,
160
+ 'maxBatchDecodeTokens',
161
+ modelId
162
+ );
158
163
 
159
164
  const batchingPatch = {
160
165
  batchSize,
@@ -196,6 +201,7 @@ function buildResolvedDecodeLoopRuntimePatch(
196
201
  readbackInterval,
197
202
  readbackMode,
198
203
  submitLatencyThresholdMs,
204
+ ...(maxBatchDecodeTokens !== undefined ? { maxBatchDecodeTokens } : {}),
199
205
  ...(ringTokens !== undefined ? { ringTokens } : {}),
200
206
  ...(ringStop !== undefined ? { ringStop } : {}),
201
207
  ...(ringStaging !== undefined ? { ringStaging } : {}),
@@ -5,7 +5,9 @@ import {
5
5
  recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
6
6
  runSiLURowSplit, recordSiLURowSplit,
7
7
  runMatmulRMSNormFused, recordMatmulRMSNormFused,
8
+ runRMSNormStats, recordRMSNormStats,
8
9
  runSandwichRMSNormPair, recordSandwichRMSNormPair,
10
+ runResidualNextRMSNormPair, recordResidualNextRMSNormPair,
9
11
  runConv2D, recordConv2D,
10
12
  } from '../../../gpu/kernel-selector.js';
11
13
  import {
@@ -58,6 +60,12 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
58
60
  return result;
59
61
  }
60
62
 
63
+ export async function doRMSNormStats(input, residual, eps, options, recorder) {
64
+ return recorder
65
+ ? await recordRMSNormStats(recorder, input, residual, eps, options)
66
+ : await runRMSNormStats(input, residual, eps, options);
67
+ }
68
+
61
69
  export async function doSandwichRMSNormPair(input, residual, postWeight, preWeight, eps, options, recorder) {
62
70
  const result = recorder
63
71
  ? await recordSandwichRMSNormPair(recorder, input, residual, postWeight, preWeight, eps, options)
@@ -73,6 +81,20 @@ export async function doSandwichRMSNormPair(input, residual, postWeight, preWeig
73
81
  return result;
74
82
  }
75
83
 
84
+ export async function doResidualNextRMSNormPair(input, residual, normWeight, eps, options = {}, recorder) {
85
+ const result = recorder
86
+ ? await recordResidualNextRMSNormPair(recorder, input, residual, normWeight, eps, options)
87
+ : await runResidualNextRMSNormPair(input, residual, normWeight, eps, options);
88
+
89
+ if (kernelTrace.enabled && !recorder) {
90
+ const layer = options.layerIdx ?? -1;
91
+ const label = options.label ?? 'residual_rmsnorm_pair';
92
+ await traceStep('residual_rmsnorm_pair.residual', label, layer, result.residual.buffer, [options.batchSize, options.hiddenSize]);
93
+ await traceStep('residual_rmsnorm_pair.next_input', label, layer, result.nextInput.buffer, [options.batchSize, options.hiddenSize]);
94
+ }
95
+
96
+ return result;
97
+ }
76
98
 
77
99
  export async function doResidualAdd(a, b, size, recorder, traceOptions) {
78
100
  const options = {};