doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -0,0 +1,208 @@
1
+ import { getRuntimeConfig } from '../../../config/runtime.js';
2
+ import { getDevice } from '../../../gpu/device.js';
3
+ import { createWeightBuffer, isWeightBuffer } from '../../../gpu/weight-buffer.js';
4
+ import { Q4K_BLOCK_BYTES, QK_K } from '../../../config/schema/index.js';
5
+
6
+ function shapeMatches(weight, rows, cols) {
7
+ return Array.isArray(weight?.shape)
8
+ && weight.shape.length === 2
9
+ && Number(weight.shape[0]) === rows
10
+ && Number(weight.shape[1]) === cols;
11
+ }
12
+
13
+ function canUseLinearAttentionABProjectionFusion({
14
+ phase,
15
+ numTokens,
16
+ debugProbes,
17
+ operatorDiagnostics,
18
+ }) {
19
+ const session = getRuntimeConfig()?.inference?.session;
20
+ if (session?.useLinearAttentionABProjectionFusion !== true) {
21
+ return false;
22
+ }
23
+ if (phase !== 'decode' || numTokens !== 1) {
24
+ return false;
25
+ }
26
+ if (debugProbes?.length || operatorDiagnostics?.enabled === true) {
27
+ return false;
28
+ }
29
+ return true;
30
+ }
31
+
32
+ export function resolveLinearAttentionABProjection(layerWeights, options) {
33
+ const {
34
+ phase,
35
+ numTokens,
36
+ hiddenSize,
37
+ numVHeads,
38
+ layerIdx,
39
+ debugProbes,
40
+ operatorDiagnostics,
41
+ } = options;
42
+ if (!canUseLinearAttentionABProjectionFusion({
43
+ phase,
44
+ numTokens,
45
+ debugProbes,
46
+ operatorDiagnostics,
47
+ })) {
48
+ return null;
49
+ }
50
+ if (layerWeights.linearABProj && shapeMatches(layerWeights.linearABProj, numVHeads * 2, hiddenSize)) {
51
+ return {
52
+ weight: layerWeights.linearABProj,
53
+ outDim: numVHeads * 2,
54
+ bProjOffsetElements: numTokens * numVHeads,
55
+ };
56
+ }
57
+
58
+ const aWeight = layerWeights.linearInProjA;
59
+ const bWeight = layerWeights.linearInProjB;
60
+ if (
61
+ !isWeightBuffer(aWeight)
62
+ || !isWeightBuffer(bWeight)
63
+ || aWeight.dtype !== 'f16'
64
+ || bWeight.dtype !== 'f16'
65
+ || aWeight.layout !== 'row'
66
+ || bWeight.layout !== 'row'
67
+ || !shapeMatches(aWeight, numVHeads, hiddenSize)
68
+ || !shapeMatches(bWeight, numVHeads, hiddenSize)
69
+ ) {
70
+ return null;
71
+ }
72
+
73
+ const bytesPerWeight = numVHeads * hiddenSize * 2;
74
+ if (aWeight.buffer.size < bytesPerWeight || bWeight.buffer.size < bytesPerWeight) {
75
+ return null;
76
+ }
77
+ const device = getDevice();
78
+ if (!device) {
79
+ return null;
80
+ }
81
+
82
+ const buffer = device.createBuffer({
83
+ label: `L${layerIdx}.linear_ab_proj_weight`,
84
+ size: bytesPerWeight * 2,
85
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
86
+ });
87
+ const encoder = device.createCommandEncoder({ label: `L${layerIdx}.linear_ab_proj_weight_pack` });
88
+ encoder.copyBufferToBuffer(aWeight.buffer, 0, buffer, 0, bytesPerWeight);
89
+ encoder.copyBufferToBuffer(bWeight.buffer, 0, buffer, bytesPerWeight, bytesPerWeight);
90
+ device.queue.submit([encoder.finish()]);
91
+
92
+ layerWeights.linearABProj = createWeightBuffer(
93
+ buffer,
94
+ 'f16',
95
+ 'row',
96
+ [numVHeads * 2, hiddenSize],
97
+ `L${layerIdx}.linear_ab_proj_weight`
98
+ );
99
+ return {
100
+ weight: layerWeights.linearABProj,
101
+ outDim: numVHeads * 2,
102
+ bProjOffsetElements: numTokens * numVHeads,
103
+ };
104
+ }
105
+
106
+ function canUseLinearAttentionQKVZProjectionFusion({
107
+ phase,
108
+ numTokens,
109
+ debugProbes,
110
+ operatorDiagnostics,
111
+ }) {
112
+ const session = getRuntimeConfig()?.inference?.session;
113
+ if (session?.useLinearAttentionQKVZProjectionFusion !== true) {
114
+ return false;
115
+ }
116
+ if (phase !== 'decode' || numTokens !== 1) {
117
+ return false;
118
+ }
119
+ if (debugProbes?.length || operatorDiagnostics?.enabled === true) {
120
+ return false;
121
+ }
122
+ return true;
123
+ }
124
+
125
+ function q4kRowBytes(hiddenSize) {
126
+ if (!Number.isInteger(hiddenSize) || hiddenSize <= 0) {
127
+ return null;
128
+ }
129
+ return Math.ceil(hiddenSize / QK_K) * Q4K_BLOCK_BYTES;
130
+ }
131
+
132
+ export function resolveLinearAttentionQKVZProjection(layerWeights, options) {
133
+ const {
134
+ phase,
135
+ numTokens,
136
+ hiddenSize,
137
+ convDim,
138
+ valueDim,
139
+ layerIdx,
140
+ debugProbes,
141
+ operatorDiagnostics,
142
+ } = options;
143
+ if (!canUseLinearAttentionQKVZProjectionFusion({
144
+ phase,
145
+ numTokens,
146
+ debugProbes,
147
+ operatorDiagnostics,
148
+ })) {
149
+ return null;
150
+ }
151
+ if (layerWeights.linearQKVZProj && shapeMatches(layerWeights.linearQKVZProj, convDim + valueDim, hiddenSize)) {
152
+ return {
153
+ weight: layerWeights.linearQKVZProj,
154
+ outDim: convDim + valueDim,
155
+ };
156
+ }
157
+
158
+ const qkvWeight = layerWeights.qkvProj;
159
+ const zWeight = layerWeights.linearInProjZ;
160
+ if (
161
+ !isWeightBuffer(qkvWeight)
162
+ || !isWeightBuffer(zWeight)
163
+ || qkvWeight.dtype !== 'q4k'
164
+ || zWeight.dtype !== 'q4k'
165
+ || qkvWeight.layout !== 'row'
166
+ || zWeight.layout !== 'row'
167
+ || !shapeMatches(qkvWeight, convDim, hiddenSize)
168
+ || !shapeMatches(zWeight, valueDim, hiddenSize)
169
+ ) {
170
+ return null;
171
+ }
172
+
173
+ const rowBytes = q4kRowBytes(hiddenSize);
174
+ if (rowBytes == null) {
175
+ return null;
176
+ }
177
+ const qkvBytes = convDim * rowBytes;
178
+ const zBytes = valueDim * rowBytes;
179
+ if (qkvWeight.buffer.size < qkvBytes || zWeight.buffer.size < zBytes) {
180
+ return null;
181
+ }
182
+ const device = getDevice();
183
+ if (!device) {
184
+ return null;
185
+ }
186
+
187
+ const buffer = device.createBuffer({
188
+ label: `L${layerIdx}.linear_qkvz_proj_weight`,
189
+ size: qkvBytes + zBytes,
190
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
191
+ });
192
+ const encoder = device.createCommandEncoder({ label: `L${layerIdx}.linear_qkvz_proj_weight_pack` });
193
+ encoder.copyBufferToBuffer(qkvWeight.buffer, 0, buffer, 0, qkvBytes);
194
+ encoder.copyBufferToBuffer(zWeight.buffer, 0, buffer, qkvBytes, zBytes);
195
+ device.queue.submit([encoder.finish()]);
196
+
197
+ layerWeights.linearQKVZProj = createWeightBuffer(
198
+ buffer,
199
+ 'q4k',
200
+ 'row',
201
+ [convDim + valueDim, hiddenSize],
202
+ `L${layerIdx}.linear_qkvz_proj_weight`
203
+ );
204
+ return {
205
+ weight: layerWeights.linearQKVZProj,
206
+ outDim: convDim + valueDim,
207
+ };
208
+ }
@@ -73,6 +73,7 @@ export interface RunLinearAttentionLayerOptions {
73
73
  weight: GPUBuffer | Float32Array | ArrayBuffer,
74
74
  label: string
75
75
  ) => GPUBuffer;
76
+ precomputedInputNorm?: Tensor | null;
76
77
  debugProbes?: ProbeConfigSchema[] | null;
77
78
  recorder?: CommandRecorder | null;
78
79
  }
@@ -1,9 +1,22 @@
1
1
  import { getBufferDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
2
- import { recordMatmul, recordRMSNorm, runMatmul, runRMSNorm, castF32ToF16, recordCastF32ToF16 } from '../../../gpu/kernel-selector.js';
2
+ import {
3
+ recordMatmul,
4
+ recordRMSNorm,
5
+ runMatmul,
6
+ runRMSNorm,
7
+ castF32ToF16,
8
+ castF16ToF32,
9
+ recordCastF32ToF16,
10
+ recordCastF16ToF32,
11
+ } from '../../../gpu/kernel-selector.js';
3
12
  import { readBuffer, releaseBuffer, uploadData, acquireBuffer } from '../../../memory/buffer-pool.js';
4
13
  import { log } from '../../../debug/index.js';
5
14
  import { decodeReadback, f16ToF32 } from './debug-utils/index.js';
6
15
  import { runLinearAttentionCoreGPU } from '../../../gpu/kernels/linear-attention-core.js';
16
+ import {
17
+ resolveLinearAttentionABProjection,
18
+ resolveLinearAttentionQKVZProjection,
19
+ } from './linear-attention-ab-fusion.js';
7
20
  import { runProbes } from './probes.js';
8
21
  import { QK_K, Q4K_BLOCK_BYTES } from '../../../config/schema/index.js';
9
22
  import { dequantizeQ4KM } from '../../../converter/quantizer.js';
@@ -656,9 +669,32 @@ async function projectLinearTensor({
656
669
  ? await recordCastF32ToF16(recorder, inputTensor)
657
670
  : await castF32ToF16(inputTensor);
658
671
  }
672
+ const settleProjectionOutputDtype = async (result) => {
673
+ if (result.dtype === resolvedOutputDtype) {
674
+ return result;
675
+ }
676
+ assertImplicitDtypeTransitionAllowed({
677
+ executionPolicies,
678
+ fromDtype: result.dtype,
679
+ toDtype: resolvedOutputDtype,
680
+ op: role,
681
+ detail: 'Linear attention projection returned a kernel-selected dtype different from the declared projection output dtype.',
682
+ });
683
+ if (resolvedOutputDtype === 'f16') {
684
+ const casted = recorder ? await recordCastF32ToF16(recorder, result) : await castF32ToF16(result);
685
+ releaseOrTrackBuffer(recorder, result.buffer);
686
+ return casted;
687
+ }
688
+ if (resolvedOutputDtype === 'f32') {
689
+ const casted = recorder ? await recordCastF16ToF32(recorder, result) : await castF16ToF32(result);
690
+ releaseOrTrackBuffer(recorder, result.buffer);
691
+ return casted;
692
+ }
693
+ throw new Error(`Unsupported linear_attention projection output dtype "${String(resolvedOutputDtype)}".`);
694
+ };
659
695
  try {
660
696
  if (recorder) {
661
- return await recordMatmul(recorder, matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
697
+ const result = await recordMatmul(recorder, matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
662
698
  transposeB: 'auto',
663
699
  role,
664
700
  layerIdx,
@@ -666,8 +702,9 @@ async function projectLinearTensor({
666
702
  outputDtype: resolvedOutputDtype,
667
703
  executionPolicies,
668
704
  });
705
+ return await settleProjectionOutputDtype(result);
669
706
  }
670
- return await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
707
+ const result = await runMatmul(matmulInput, resolvedWeight, numTokens, outDim, hiddenSize, {
671
708
  transposeB: 'auto',
672
709
  role,
673
710
  layerIdx,
@@ -675,6 +712,7 @@ async function projectLinearTensor({
675
712
  outputDtype: resolvedOutputDtype,
676
713
  executionPolicies,
677
714
  });
715
+ return await settleProjectionOutputDtype(result);
678
716
  } finally {
679
717
  if (matmulInput !== inputTensor) {
680
718
  releaseOrTrackBuffer(recorder, matmulInput.buffer);
@@ -683,6 +721,31 @@ async function projectLinearTensor({
683
721
  }
684
722
  }
685
723
 
724
+ async function settleLinearAttentionCoreInputDtype(tensor, targetDtype, options) {
725
+ if (tensor.dtype === targetDtype) {
726
+ return tensor;
727
+ }
728
+ const {
729
+ recorder,
730
+ executionPolicies,
731
+ role,
732
+ } = options;
733
+ assertImplicitDtypeTransitionAllowed({
734
+ executionPolicies,
735
+ fromDtype: tensor.dtype,
736
+ toDtype: targetDtype,
737
+ op: role,
738
+ detail: 'Linear attention core requires qkv/z/a/b tensors to share one input dtype.',
739
+ });
740
+ if (targetDtype === 'f16') {
741
+ return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
742
+ }
743
+ if (targetDtype === 'f32') {
744
+ return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
745
+ }
746
+ throw new Error(`Unsupported linear_attention core input dtype "${String(targetDtype)}".`);
747
+ }
748
+
686
749
  export function hasLinearAttentionLayers(layerTypes) {
687
750
  if (!Array.isArray(layerTypes)) return false;
688
751
  for (let i = 0; i < layerTypes.length; i++) {
@@ -771,6 +834,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
771
834
  getNormWeightBuffer,
772
835
  recorder,
773
836
  executionPolicies = null,
837
+ precomputedInputNorm = null,
774
838
  } = options;
775
839
 
776
840
  if (!layerWeights) {
@@ -808,7 +872,20 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
808
872
  let normedTensor = inputTensor;
809
873
  let normedCreated = false;
810
874
 
811
- if (layerWeights.inputNorm) {
875
+ if (precomputedInputNorm) {
876
+ if (
877
+ phase !== 'decode'
878
+ || numTokens !== 1
879
+ || !layerWeights.inputNorm
880
+ || precomputedInputNorm.dtype !== inputTensor.dtype
881
+ || precomputedInputNorm.dtype !== 'f32'
882
+ ) {
883
+ releaseOrTrackBuffer(recorder, precomputedInputNorm.buffer);
884
+ throw new Error(`linear_attention layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
885
+ }
886
+ normedTensor = precomputedInputNorm;
887
+ normedCreated = true;
888
+ } else if (layerWeights.inputNorm) {
812
889
  const normWeightBuffer = getNormWeightBuffer(layerWeights.inputNorm, `L${layerIdx}.linear_input_norm`);
813
890
  try {
814
891
  if (recorder) {
@@ -832,7 +909,35 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
832
909
  }
833
910
  }
834
911
 
835
- const qkvTensor = await projectLinearTensor({
912
+ const qkvzProjection = resolveLinearAttentionQKVZProjection(layerWeights, {
913
+ phase,
914
+ numTokens,
915
+ hiddenSize,
916
+ numVHeads: projectionLayout.numVHeads,
917
+ convDim: projectionLayout.convDim,
918
+ valueDim: projectionLayout.valueDim,
919
+ layerIdx,
920
+ debugProbes: options.debugProbes,
921
+ operatorDiagnostics: options.operatorDiagnostics,
922
+ });
923
+ const qkvzTensor = qkvzProjection
924
+ ? await projectLinearTensor({
925
+ inputTensor: normedTensor,
926
+ sourceWeight: qkvzProjection.weight,
927
+ role: 'linear_qkvz_proj',
928
+ phase,
929
+ outDim: qkvzProjection.outDim,
930
+ numTokens,
931
+ hiddenSize,
932
+ layerIdx,
933
+ kernelPath,
934
+ outputDtype: projectionDtype,
935
+ getWeightBuffer,
936
+ recorder,
937
+ executionPolicies,
938
+ })
939
+ : null;
940
+ const qkvTensor = qkvzTensor ?? await projectLinearTensor({
836
941
  inputTensor: normedTensor,
837
942
  sourceWeight: layerWeights.qkvProj,
838
943
  role: 'linear_qkv_proj',
@@ -847,7 +952,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
847
952
  recorder,
848
953
  executionPolicies,
849
954
  });
850
- const zTensor = await projectLinearTensor({
955
+ const zTensor = qkvzTensor ?? await projectLinearTensor({
851
956
  inputTensor: normedTensor,
852
957
  sourceWeight: layerWeights.linearInProjZ,
853
958
  role: 'linear_z_proj',
@@ -862,7 +967,33 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
862
967
  recorder,
863
968
  executionPolicies,
864
969
  });
865
- const aTensor = await projectLinearTensor({
970
+ const abProjection = resolveLinearAttentionABProjection(layerWeights, {
971
+ phase,
972
+ numTokens,
973
+ hiddenSize,
974
+ numVHeads: projectionLayout.numVHeads,
975
+ layerIdx,
976
+ debugProbes: options.debugProbes,
977
+ operatorDiagnostics: options.operatorDiagnostics,
978
+ });
979
+ const abTensor = abProjection
980
+ ? await projectLinearTensor({
981
+ inputTensor: normedTensor,
982
+ sourceWeight: abProjection.weight,
983
+ role: 'linear_ab_proj',
984
+ phase,
985
+ outDim: abProjection.outDim,
986
+ numTokens,
987
+ hiddenSize,
988
+ layerIdx,
989
+ kernelPath,
990
+ outputDtype: projectionDtype,
991
+ getWeightBuffer,
992
+ recorder,
993
+ executionPolicies,
994
+ })
995
+ : null;
996
+ const aTensor = abTensor ?? await projectLinearTensor({
866
997
  inputTensor: normedTensor,
867
998
  sourceWeight: layerWeights.linearInProjA,
868
999
  role: 'linear_a_proj',
@@ -877,7 +1008,7 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
877
1008
  recorder,
878
1009
  executionPolicies,
879
1010
  });
880
- const bTensor = await projectLinearTensor({
1011
+ const bTensor = abTensor ?? await projectLinearTensor({
881
1012
  inputTensor: normedTensor,
882
1013
  sourceWeight: layerWeights.linearInProjB,
883
1014
  role: 'linear_b_proj',
@@ -909,6 +1040,10 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
909
1040
  layerOutputDtype,
910
1041
  'outputDtype'
911
1042
  );
1043
+ let coreQkvTensor = qkvTensor;
1044
+ let coreZTensor = zTensor;
1045
+ let coreATensor = aTensor;
1046
+ let coreBTensor = bTensor;
912
1047
 
913
1048
  try {
914
1049
  await runProbes('linear_qkv_proj', qkvTensor.buffer, {
@@ -947,11 +1082,27 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
947
1082
  operatorDiagnostics: options.operatorDiagnostics,
948
1083
  dtype: bTensor.dtype,
949
1084
  });
1085
+ const coreInputDtype = qkvTensor.dtype;
1086
+ coreZTensor = await settleLinearAttentionCoreInputDtype(zTensor, coreInputDtype, {
1087
+ recorder,
1088
+ executionPolicies,
1089
+ role: 'linear_z_proj',
1090
+ });
1091
+ coreATensor = await settleLinearAttentionCoreInputDtype(aTensor, coreInputDtype, {
1092
+ recorder,
1093
+ executionPolicies,
1094
+ role: 'linear_a_proj',
1095
+ });
1096
+ coreBTensor = await settleLinearAttentionCoreInputDtype(bTensor, coreInputDtype, {
1097
+ recorder,
1098
+ executionPolicies,
1099
+ role: 'linear_b_proj',
1100
+ });
950
1101
  const coreTensor = await runLinearAttentionCoreGPU(
951
- qkvTensor,
952
- zTensor,
953
- aTensor,
954
- bTensor,
1102
+ coreQkvTensor,
1103
+ coreZTensor,
1104
+ coreATensor,
1105
+ coreBTensor,
955
1106
  layerState,
956
1107
  {
957
1108
  numTokens,
@@ -960,6 +1111,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
960
1111
  qkL2NormEps: QK_L2NORM_EPS,
961
1112
  recorder,
962
1113
  executionPolicies,
1114
+ abPacked: abProjection != null,
1115
+ qkvzPacked: qkvzProjection != null,
1116
+ bProjOffsetElements: abProjection?.bProjOffsetElements ?? 0,
963
1117
  }
964
1118
  );
965
1119
  await runProbes('linear_core_out', coreTensor.buffer, {
@@ -1017,9 +1171,12 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
1017
1171
  if (normedCreated) {
1018
1172
  releaseOrTrackBuffer(recorder, normedTensor.buffer);
1019
1173
  }
1020
- releaseOrTrackBuffer(recorder, qkvTensor.buffer);
1021
- releaseOrTrackBuffer(recorder, zTensor.buffer);
1022
- releaseOrTrackBuffer(recorder, aTensor.buffer);
1023
- releaseOrTrackBuffer(recorder, bTensor.buffer);
1174
+ const released = new Set();
1175
+ for (const tensor of [qkvTensor, zTensor, aTensor, bTensor, coreQkvTensor, coreZTensor, coreATensor, coreBTensor]) {
1176
+ const buffer = tensor?.buffer ?? null;
1177
+ if (!buffer || released.has(buffer)) continue;
1178
+ released.add(buffer);
1179
+ releaseOrTrackBuffer(recorder, buffer);
1180
+ }
1024
1181
  }
1025
1182
  }
@@ -5,7 +5,7 @@ import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js'
5
5
  import { runMatmul, runRMSNorm, runScale, recordScale } from '../../../../gpu/kernel-selector.js';
6
6
  import { recordMatmul } from '../../../../gpu/kernels/matmul.js';
7
7
  import { recordRMSNorm } from '../../../../gpu/kernels/rmsnorm.js';
8
- import { recordLmHeadArgmaxF16 } from '../../../../gpu/kernels/lm-head-argmax.js';
8
+ import { recordLmHeadArgmax } from '../../../../gpu/kernels/lm-head-argmax.js';
9
9
  import { createTensor } from '../../../../gpu/tensor.js';
10
10
  import {
11
11
  castF16ToF32,
@@ -1176,7 +1176,7 @@ export async function recordGreedyLmHeadArgmaxGPU(
1176
1176
  config,
1177
1177
  operatorDiagnostics
1178
1178
  );
1179
- const outputBuffer = await recordLmHeadArgmaxF16(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1179
+ const outputBuffer = await recordLmHeadArgmax(recorder, tail.lmHeadInputTensor, tail.lmHeadBuffer, {
1180
1180
  vocabSize: tail.matmulVocabSize,
1181
1181
  hiddenSize: tail.hiddenSize,
1182
1182
  padTokenId: options.padTokenId,
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
12
12
  // Imports for computeLogits orchestrator
13
13
  import { getDevice } from '../../../../gpu/device.js';
14
14
  import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
15
- import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16 } from '../../../../gpu/kernel-selector.js';
15
+ import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
16
16
  import { createTensor } from '../../../../gpu/tensor.js';
17
17
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
18
18
  import { kernelTrace, traceStep } from '../kernel-trace.js';
@@ -50,6 +50,28 @@ function resolveLmHeadMatmulRole(phase) {
50
50
  return phase === 'prefill' ? 'lm_head_prefill' : 'lm_head';
51
51
  }
52
52
 
53
+ function normalizeSelectedLogitTokenIds(value, vocabSize) {
54
+ if (value == null) {
55
+ return null;
56
+ }
57
+ if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
58
+ throw new Error('[Logits] selectedTokenIds must be an array or typed array.');
59
+ }
60
+ const tokenIds = Array.from(value, (entry, index) => {
61
+ const tokenId = Number(entry);
62
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
63
+ throw new Error(
64
+ `[Logits] selectedTokenIds[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`
65
+ );
66
+ }
67
+ return tokenId;
68
+ });
69
+ if (tokenIds.length === 0) {
70
+ throw new Error('[Logits] selectedTokenIds must not be empty.');
71
+ }
72
+ return tokenIds;
73
+ }
74
+
53
75
  async function coerceTensorDtype(tensor, targetDtype, options = {}) {
54
76
  if (!targetDtype || tensor.dtype === targetDtype) {
55
77
  return tensor;
@@ -196,6 +218,7 @@ export async function computeLogits(
196
218
  matmulVocabSize = dims.vocabSize;
197
219
  }
198
220
  }
221
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(options?.selectedTokenIds, matmulVocabSize);
199
222
 
200
223
  // Check if input is GPU buffer
201
224
  const inputIsGPU = isGpuBufferInstance(hiddenStates);
@@ -204,6 +227,12 @@ export async function computeLogits(
204
227
  if (isTraceEnabled('logits')) {
205
228
  trace.logits(`LOGITS_PATH: device=${!!device}, useGPU=${useGPU}, taking ${(!device || !useGPU) ? 'CPU' : 'GPU'} path`);
206
229
  }
230
+ if (selectedTokenIds && (!device || !useGPU)) {
231
+ throw new Error('[Logits] selectedTokenIds requires the GPU logits path.');
232
+ }
233
+ if (selectedTokenIds && (isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
234
+ throw new Error('[Logits] selectedTokenIds requires a single GPU-resident LM head weight buffer.');
235
+ }
207
236
  if (!device || !useGPU) {
208
237
 
209
238
  let cpuHiddenStates;
@@ -481,6 +510,38 @@ export async function computeLogits(
481
510
  matmulInputOwned = true;
482
511
  }
483
512
 
513
+ if (selectedTokenIds) {
514
+ if (lastPositionOnly !== true) {
515
+ throw new Error('[Logits] selectedTokenIds requires lastPositionOnly=true.');
516
+ }
517
+ const selected = await runLmHeadSelectLogitsF16(matmulInputTensor, lmHeadBuffer, {
518
+ device,
519
+ hiddenSize,
520
+ vocabSize: matmulVocabSize,
521
+ tokenIds: selectedTokenIds,
522
+ hiddenOffset: 0,
523
+ logitSoftcap: config.finalLogitSoftcapping == null ? 0 : Number(config.finalLogitSoftcapping),
524
+ });
525
+ const selectedBytes = selectedTokenIds.length * Float32Array.BYTES_PER_ELEMENT;
526
+ const selectedData = await readBufferWithCleanup(selected.outputBuffer, selectedBytes, () => {
527
+ releaseBuffer(selected.outputBuffer);
528
+ releaseBuffer(selected.tokenIdBuffer);
529
+ if (inputBufferOwned) releaseBuffer(inputBuffer);
530
+ releaseBuffer(finalNormTensor.buffer);
531
+ if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
532
+ if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
533
+ if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
534
+ });
535
+ const selectedLogits = new Float32Array(selectedData);
536
+ await runProbes('logits_final', selectedLogits, {
537
+ numTokens: 1,
538
+ hiddenSize: selectedTokenIds.length,
539
+ probes: debugProbes,
540
+ operatorDiagnostics,
541
+ });
542
+ return selectedLogits;
543
+ }
544
+
484
545
  // HuggingFace models store lm_head as [vocabSize, hiddenSize], so transposeB=true
485
546
  const logitsTensor = await runMatmul(matmulInputTensor, lmHeadBuffer, matmulRows, matmulVocabSize, hiddenSize, {
486
547
  transposeB: 'auto',
@@ -46,12 +46,12 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
46
46
  let scaled = null;
47
47
  try {
48
48
  loraIntermediate = recorder
49
- ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
- : await runMatmul(input, aBuf, M, rank, K, { transposeB: 'auto', role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
49
+ ? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
50
+ : await runMatmul(input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
51
51
 
52
52
  loraOutput = recorder
53
- ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype })
54
- : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: 'auto', role: 'lora_b', kernelPath, outputDtype });
53
+ ? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype })
54
+ : await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype });
55
55
 
56
56
  scaled = recorder
57
57
  ? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
@@ -155,6 +155,11 @@ function buildResolvedDecodeLoopRuntimePatch(
155
155
  );
156
156
  }
157
157
  const submitLatencyThresholdMs = decodeLoop.submitLatencyThresholdMs ?? null;
158
+ const maxBatchDecodeTokens = parseSessionDecodeLoopOptionalPositiveInt(
159
+ decodeLoop.maxBatchDecodeTokens,
160
+ 'maxBatchDecodeTokens',
161
+ modelId
162
+ );
158
163
 
159
164
  const batchingPatch = {
160
165
  batchSize,
@@ -196,6 +201,7 @@ function buildResolvedDecodeLoopRuntimePatch(
196
201
  readbackInterval,
197
202
  readbackMode,
198
203
  submitLatencyThresholdMs,
204
+ ...(maxBatchDecodeTokens !== undefined ? { maxBatchDecodeTokens } : {}),
199
205
  ...(ringTokens !== undefined ? { ringTokens } : {}),
200
206
  ...(ringStop !== undefined ? { ringStop } : {}),
201
207
  ...(ringStaging !== undefined ? { ringStaging } : {}),
@@ -122,6 +122,7 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
122
122
  activatedOutput = await activationFn(upOutput, {
123
123
  size: numTokens * intermediateSize,
124
124
  gate: gateOutput,
125
+ inputActivation: 'identity',
125
126
  swigluLimit,
126
127
  });
127
128
 
@@ -1117,6 +1117,7 @@ async function runMixtralExpert(
1117
1117
  const activated = await activationFn(upOut, {
1118
1118
  size: count * intermediateSize,
1119
1119
  gate: gateOut,
1120
+ inputActivation: 'identity',
1120
1121
  swigluLimit,
1121
1122
  });
1122
1123
  releaseBuffer(gateOut.buffer);