doppler-gpu 0.4.7 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1589 -0
  4. package/package.json +44 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  16. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  17. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  18. package/src/config/kernel-path-loader.js +1 -0
  19. package/src/config/kernels/backward-registry.json +13 -2
  20. package/src/config/kernels/kernel-ref-digests.js +34 -21
  21. package/src/config/kernels/registry.json +1402 -2623
  22. package/src/config/merge.js +11 -0
  23. package/src/config/param-validator.js +25 -0
  24. package/src/config/runtime/default.json +3 -1
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  29. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  30. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  31. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  32. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  33. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  34. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  35. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  36. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  37. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  38. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  39. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  40. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  41. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  42. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  43. package/src/config/schema/doppler.schema.js +40 -5
  44. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  45. package/src/config/schema/execution-v1.schema.js +10 -0
  46. package/src/config/schema/index.d.ts +3 -1
  47. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  48. package/src/config/schema/training-metrics.schema.js +4 -0
  49. package/src/config/transforms/capability-transform-resolver.js +7 -1
  50. package/src/config/transforms/execution-graph-transforms.js +2 -2
  51. package/src/debug/config.d.ts +4 -0
  52. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  53. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  54. package/src/experimental/orchestration/index.d.ts +10 -1
  55. package/src/experimental/training/attention-backward.js +123 -55
  56. package/src/experimental/training/autograd.js +101 -5
  57. package/src/experimental/training/clip.js +6 -1
  58. package/src/experimental/training/datasets/text-pairs.js +80 -8
  59. package/src/experimental/training/distillation/student-fixture.js +470 -114
  60. package/src/experimental/training/lora-pipeline.js +202 -27
  61. package/src/experimental/training/lora.js +10 -4
  62. package/src/experimental/training/optimizer.js +9 -1
  63. package/src/experimental/training/runner.js +18 -4
  64. package/src/experimental/training/suite.js +1 -1
  65. package/src/experimental/training/workloads.js +11 -0
  66. package/src/formats/rdrr/classification.js +3 -0
  67. package/src/formats/rdrr/integrity.d.ts +0 -2
  68. package/src/formats/rdrr/integrity.js +0 -2
  69. package/src/formats/rdrr/types.d.ts +1 -1
  70. package/src/gpu/kernels/attention.d.ts +2 -0
  71. package/src/gpu/kernels/attention.js +75 -16
  72. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  73. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  74. package/src/gpu/kernels/backward/adam.js +6 -4
  75. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  76. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  77. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  78. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  79. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  80. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  81. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  82. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  83. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  84. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  85. package/src/gpu/kernels/backward/index.d.ts +3 -0
  86. package/src/gpu/kernels/backward/index.js +3 -0
  87. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  88. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  89. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  90. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  91. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  92. package/src/gpu/kernels/backward/utils.js +9 -2
  93. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  94. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  95. package/src/gpu/kernels/fused_ffn.js +540 -19
  96. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  97. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  98. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  99. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  100. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  101. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  102. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  103. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  104. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  105. package/src/gpu/kernels/index.d.ts +26 -0
  106. package/src/gpu/kernels/index.js +18 -0
  107. package/src/gpu/kernels/linear-attention-core.js +413 -15
  108. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  109. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  110. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  111. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  112. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  113. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  114. package/src/gpu/kernels/matmul-selection.js +10 -1
  115. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  116. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  117. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  118. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  119. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  120. package/src/gpu/kernels/silu.d.ts +1 -0
  121. package/src/gpu/kernels/silu.js +19 -4
  122. package/src/gpu/kernels/silu.wgsl +7 -1
  123. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  124. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  125. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  126. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  127. package/src/gpu/tensor.d.ts +1 -0
  128. package/src/index.d.ts +1 -1
  129. package/src/index.js +1 -1
  130. package/src/inference/browser-harness-suite-helpers.js +1 -1
  131. package/src/inference/browser-harness-text-helpers.js +251 -20
  132. package/src/inference/browser-harness.js +221 -3
  133. package/src/inference/kv-cache/base.js +42 -1
  134. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  135. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  136. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  137. package/src/inference/pipelines/text/attention/projections.js +174 -4
  138. package/src/inference/pipelines/text/attention/record.js +44 -21
  139. package/src/inference/pipelines/text/attention/run.js +18 -3
  140. package/src/inference/pipelines/text/config.d.ts +18 -0
  141. package/src/inference/pipelines/text/config.js +100 -0
  142. package/src/inference/pipelines/text/embed.js +1 -1
  143. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  144. package/src/inference/pipelines/text/execution-plan.js +37 -0
  145. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  146. package/src/inference/pipelines/text/execution-v1.js +130 -10
  147. package/src/inference/pipelines/text/ffn/dense.js +241 -21
  148. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  149. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  152. package/src/inference/pipelines/text/generator-steps.js +2 -2
  153. package/src/inference/pipelines/text/generator.d.ts +18 -1
  154. package/src/inference/pipelines/text/generator.js +326 -19
  155. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  156. package/src/inference/pipelines/text/init.d.ts +22 -3
  157. package/src/inference/pipelines/text/init.js +2 -2
  158. package/src/inference/pipelines/text/layer.js +87 -8
  159. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  160. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  161. package/src/inference/pipelines/text/linear-attention.js +173 -16
  162. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  163. package/src/inference/pipelines/text/logits/index.js +62 -1
  164. package/src/inference/pipelines/text/lora-apply.js +4 -4
  165. package/src/inference/pipelines/text/model-load.js +6 -0
  166. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  167. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  168. package/src/inference/pipelines/text/ops.js +22 -0
  169. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  170. package/src/inference/pipelines/text/probes.js +9 -4
  171. package/src/inference/pipelines/text/state.d.ts +4 -4
  172. package/src/inference/pipelines/text/state.js +4 -0
  173. package/src/inference/pipelines/text/types.d.ts +27 -2
  174. package/src/inference/pipelines/text.d.ts +21 -1
  175. package/src/inference/pipelines/text.js +40 -14
  176. package/src/inference/tokenizers/bundled.js +150 -55
  177. package/src/inference/tokenizers/types.d.ts +7 -4
  178. package/src/loader/doppler-loader.js +25 -2
  179. package/src/loader/index.d.ts +3 -1
  180. package/src/loader/index.js +2 -0
  181. package/src/loader/loader-types.d.ts +3 -1
  182. package/src/loader/manifest-config.d.ts +18 -0
  183. package/src/loader/manifest-config.js +11 -3
  184. package/src/loader/shard-cache.d.ts +2 -0
  185. package/src/loader/shard-cache.js +29 -1
  186. package/src/models/gemma4.js +8 -0
  187. package/src/rules/inference/capability-transforms.rules.json +145 -20
  188. package/src/rules/inference/ffn.rules.json +33 -0
  189. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  190. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  191. package/src/rules/kernels/matmul.rules.json +2 -0
  192. package/src/rules/kernels/silu.rules.json +1 -0
  193. package/src/rules/rule-registry.js +2 -0
  194. package/src/storage/artifact-storage-context.js +36 -0
  195. package/src/tooling/node-browser-command-runner.js +42 -7
  196. package/src/tooling/node-converter.js +20 -5
  197. package/src/tooling/opfs-cache.d.ts +21 -1
  198. package/src/tooling/opfs-cache.js +168 -32
  199. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  200. package/src/tooling-experimental-exports.shared.js +18 -0
  201. package/src/tooling-exports/storage.d.ts +1 -1
  202. package/src/tooling-exports/storage.js +1 -1
  203. package/src/types/gpu.d.ts +25 -0
  204. package/src/version.js +1 -1
  205. package/tests/kernels/browser/kernel-suite.js +1593 -0
  206. package/tests/kernels/browser/test-page.js +2629 -0
  207. package/tests/kernels/harness/benchmark.js +225 -0
  208. package/tests/kernels/harness/buffer-utils.js +88 -0
  209. package/tests/kernels/harness/tolerance.js +174 -0
  210. package/tests/kernels/reference/adam.js +15 -0
  211. package/tests/kernels/reference/attention.js +158 -0
  212. package/tests/kernels/reference/backward.js +96 -0
  213. package/tests/kernels/reference/bias-add.js +20 -0
  214. package/tests/kernels/reference/conv2d.js +58 -0
  215. package/tests/kernels/reference/cross-entropy.js +26 -0
  216. package/tests/kernels/reference/dequant.js +312 -0
  217. package/tests/kernels/reference/embed.js +15 -0
  218. package/tests/kernels/reference/gather.js +61 -0
  219. package/tests/kernels/reference/gelu.js +37 -0
  220. package/tests/kernels/reference/groupnorm.js +36 -0
  221. package/tests/kernels/reference/index.js +89 -0
  222. package/tests/kernels/reference/layernorm.js +22 -0
  223. package/tests/kernels/reference/matmul.js +57 -0
  224. package/tests/kernels/reference/modulate.js +24 -0
  225. package/tests/kernels/reference/moe-gather.js +83 -0
  226. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  227. package/tests/kernels/reference/residual.js +55 -0
  228. package/tests/kernels/reference/rmsnorm.js +52 -0
  229. package/tests/kernels/reference/rope.js +79 -0
  230. package/tests/kernels/reference/sample.js +97 -0
  231. package/tests/kernels/reference/scatter-add.js +58 -0
  232. package/tests/kernels/reference/silu.js +52 -0
  233. package/tests/kernels/reference/softmax.js +86 -0
  234. package/tests/kernels/reference/split-qkv.js +58 -0
  235. package/tests/kernels/reference/topk.js +88 -0
  236. package/tests/kernels/reference/upsample2d.js +37 -0
  237. package/BRANDING.md +0 -14
  238. package/SECURITY.md +0 -19
  239. package/assets/doppler-runtime-map.svg +0 -82
  240. package/src/bootstrap.js +0 -30
  241. package/src/cli/doppler-cli.d.ts +0 -46
  242. package/src/cli/doppler-quickstart.d.ts +0 -45
  243. package/src/cli/doppler-serve.d.ts +0 -210
  244. package/src/cli/runtime-profiles.d.ts +0 -40
  245. package/src/client/doppler-api.browser.d.ts +0 -1
  246. package/src/client/doppler-api.browser.js +0 -25
  247. package/src/client/doppler-registry.d.ts +0 -54
  248. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  249. package/src/client/runtime/source-runtime.d.ts +0 -26
  250. package/src/client/runtime/storage.d.ts +0 -5
  251. package/src/config/README.md +0 -61
  252. package/src/config/conversion/README.md +0 -62
  253. package/src/config/goal-completion-matrix.json +0 -552
  254. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  255. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  256. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  257. package/src/config/kernels/registry.d.ts +0 -51
  258. package/src/config/merge-contract-check.d.ts +0 -16
  259. package/src/config/merge-contract-check.js +0 -319
  260. package/src/config/merge-helpers.d.ts +0 -60
  261. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  262. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  263. package/src/config/param-categories.d.ts +0 -8
  264. package/src/config/param-validator.d.ts +0 -33
  265. package/src/config/platforms/loader.d.ts +0 -38
  266. package/src/config/quantization-contract-check.d.ts +0 -12
  267. package/src/config/quantization-contract-check.js +0 -91
  268. package/src/config/runtime/README.md +0 -28
  269. package/src/config/runtime-merge.d.ts +0 -5
  270. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  271. package/src/config/schema/command-surface-contract.schema.json +0 -81
  272. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  273. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  274. package/src/config/schema/platform.schema.js +0 -1
  275. package/src/config/schema/policy-schema-registry.json +0 -31
  276. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  277. package/src/config/support-tiers/subsystems.json +0 -502
  278. package/src/config/supported-operations.d.ts +0 -14
  279. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  280. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  281. package/src/converter/conversion-plan.d.ts +0 -56
  282. package/src/converter/index.d.ts +0 -98
  283. package/src/converter/index.js +0 -59
  284. package/src/converter/parsers/diffusion.d.ts +0 -50
  285. package/src/converter/parsers/gguf.d.ts +0 -31
  286. package/src/converter/parsers/index.d.ts +0 -21
  287. package/src/converter/parsers/index.js +0 -12
  288. package/src/converter/parsers/transformer.d.ts +0 -31
  289. package/src/converter/quantization-info.d.ts +0 -38
  290. package/src/converter/rope-config.d.ts +0 -16
  291. package/src/converter/tokenizer-utils.d.ts +0 -12
  292. package/src/debug/reference/README.md +0 -114
  293. package/src/debug/reference/hf_attn_debug.py +0 -114
  294. package/src/debug/reference/hf_embed_check.py +0 -89
  295. package/src/debug/reference/hf_layer_out.py +0 -100
  296. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  297. package/src/debug/reference/hf_rope_check.py +0 -116
  298. package/src/debug/reference/hf_weights.py +0 -75
  299. package/src/errors/doppler-error.d.ts +0 -20
  300. package/src/experimental/bridge/extension/background.d.ts +0 -14
  301. package/src/experimental/bridge/extension/background.js +0 -168
  302. package/src/experimental/bridge/extension/manifest.json +0 -34
  303. package/src/experimental/bridge/extension-client.d.ts +0 -114
  304. package/src/experimental/bridge/index.d.ts +0 -69
  305. package/src/experimental/bridge/node-bridge.js +0 -24
  306. package/src/experimental/bridge/protocol.d.ts +0 -96
  307. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  308. package/src/experimental/browser/gguf-importer.js +0 -532
  309. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  310. package/src/experimental/browser/quantization.d.ts +0 -74
  311. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  312. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  313. package/src/experimental/browser/shard-io-browser.js +0 -89
  314. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  315. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  316. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  317. package/src/experimental/distribution/p2p-transport.js +0 -75
  318. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  319. package/src/experimental/hotswap/manifest.d.ts +0 -42
  320. package/src/experimental/hotswap/model-swap.js +0 -42
  321. package/src/experimental/hotswap/runtime.d.ts +0 -31
  322. package/src/experimental/training/README.md +0 -164
  323. package/src/experimental/training/artifacts.d.ts +0 -160
  324. package/src/experimental/training/attention-backward.d.ts +0 -30
  325. package/src/experimental/training/attention-forward.d.ts +0 -22
  326. package/src/experimental/training/attention-forward.js +0 -82
  327. package/src/experimental/training/autograd.d.ts +0 -51
  328. package/src/experimental/training/backward-kernels.js +0 -39
  329. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  330. package/src/experimental/training/checkpoint.d.ts +0 -36
  331. package/src/experimental/training/clip.d.ts +0 -9
  332. package/src/experimental/training/dataloader.d.ts +0 -8
  333. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  334. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  335. package/src/experimental/training/datasets/index.d.ts +0 -24
  336. package/src/experimental/training/datasets/index.js +0 -13
  337. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  338. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  339. package/src/experimental/training/datasets/reploid.js +0 -36
  340. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  341. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  342. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  343. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  344. package/src/experimental/training/datasets/token-batch.js +0 -52
  345. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  346. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  347. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  348. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  349. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  350. package/src/experimental/training/distillation/eval.d.ts +0 -34
  351. package/src/experimental/training/distillation/index.d.ts +0 -29
  352. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  353. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  354. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  355. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  356. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  357. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  358. package/src/experimental/training/export.d.ts +0 -41
  359. package/src/experimental/training/index.d.ts +0 -74
  360. package/src/experimental/training/index.js +0 -63
  361. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  362. package/src/experimental/training/lora.d.ts +0 -19
  363. package/src/experimental/training/loss-scaling.d.ts +0 -21
  364. package/src/experimental/training/loss.d.ts +0 -10
  365. package/src/experimental/training/objectives/base.d.ts +0 -58
  366. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  367. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  368. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  369. package/src/experimental/training/objectives/index.d.ts +0 -12
  370. package/src/experimental/training/objectives/index.js +0 -6
  371. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  372. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  373. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  374. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  375. package/src/experimental/training/operator-command.d.ts +0 -5
  376. package/src/experimental/training/operator-eval.d.ts +0 -48
  377. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  378. package/src/experimental/training/optimizer.d.ts +0 -22
  379. package/src/experimental/training/runner.d.ts +0 -249
  380. package/src/experimental/training/suite.d.ts +0 -299
  381. package/src/experimental/training/tensor-factory.d.ts +0 -9
  382. package/src/experimental/training/trainer.d.ts +0 -89
  383. package/src/experimental/training/ul_dataset.d.ts +0 -47
  384. package/src/experimental/training/ul_schedule.d.ts +0 -6
  385. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  386. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  387. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  388. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  389. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  390. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  391. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  392. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  393. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  394. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  395. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  396. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  397. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  398. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  399. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  400. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  401. package/src/experimental/training/workload-packs/registry.json +0 -278
  402. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  403. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  404. package/src/experimental/training/workloads.d.ts +0 -201
  405. package/src/formats/gguf/types.d.ts +0 -74
  406. package/src/formats/index.d.ts +0 -75
  407. package/src/formats/index.js +0 -19
  408. package/src/formats/litert/types.d.ts +0 -74
  409. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  410. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  411. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  412. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  413. package/src/formats/safetensors/types.d.ts +0 -67
  414. package/src/formats/tflite/types.d.ts +0 -61
  415. package/src/formats/tokenizer/index.d.ts +0 -5
  416. package/src/formats/tokenizer/index.js +0 -3
  417. package/src/formats/tokenizer/types.d.ts +0 -9
  418. package/src/gpu/kernel-runtime.d.ts +0 -13
  419. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  420. package/src/gpu/kernel-selection-log.d.ts +0 -4
  421. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  422. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  423. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  424. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  425. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  426. package/src/gpu/kernel-tuner/types.js +0 -4
  427. package/src/gpu/kernel-tuner.d.ts +0 -33
  428. package/src/gpu/kernels/README.md +0 -166
  429. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  430. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  431. package/src/gpu/kernels/check-stop.d.ts +0 -31
  432. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  433. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  434. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  435. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  436. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  437. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  438. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  439. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  440. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  441. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  442. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  443. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  444. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  445. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  446. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  447. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  448. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  449. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  450. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  451. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  452. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  453. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  454. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  455. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  456. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  457. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  458. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  459. package/src/gpu/kernels/constants.d.ts +0 -129
  460. package/src/gpu/kernels/dispatch.d.ts +0 -72
  461. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  462. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  463. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  464. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  465. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  466. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  467. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  468. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  469. package/src/gpu/kernels/types.js +0 -4
  470. package/src/gpu/perf-guards.d.ts +0 -13
  471. package/src/gpu/profiler.d.ts +0 -93
  472. package/src/gpu/readback-utils.d.ts +0 -16
  473. package/src/gpu/readback-utils.js +0 -49
  474. package/src/gpu/submit-probe.d.ts +0 -12
  475. package/src/index-browser.d.ts +0 -2
  476. package/src/index-browser.js +0 -2
  477. package/src/inference/README.md +0 -593
  478. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  479. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  480. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  481. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  482. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  483. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  484. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  485. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  486. package/src/inference/network-evolution.js +0 -79
  487. package/src/inference/pipelines/context.d.ts +0 -21
  488. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  489. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  490. package/src/inference/pipelines/diffusion/index.js +0 -3
  491. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  492. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  493. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  494. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  495. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  496. package/src/inference/pipelines/diffusion/types.js +0 -1
  497. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  498. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  499. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  500. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  501. package/src/inference/pipelines/dream/pipeline.js +0 -8
  502. package/src/inference/pipelines/energy/index.d.ts +0 -1
  503. package/src/inference/pipelines/energy/index.js +0 -1
  504. package/src/inference/pipelines/energy/types.js +0 -1
  505. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  506. package/src/inference/pipelines/energy-head/index.js +0 -6
  507. package/src/inference/pipelines/factory.d.ts +0 -10
  508. package/src/inference/pipelines/index.d.ts +0 -22
  509. package/src/inference/pipelines/index.js +0 -19
  510. package/src/inference/pipelines/registry.d.ts +0 -15
  511. package/src/inference/pipelines/rng.d.ts +0 -2
  512. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  513. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  514. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  515. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  516. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  517. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  518. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  519. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  520. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  521. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  522. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  523. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  524. package/src/inference/pipelines/text/attention.d.ts +0 -23
  525. package/src/inference/pipelines/text/attention.js +0 -12
  526. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  527. package/src/inference/pipelines/text/buffer-types.js +0 -4
  528. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  529. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  530. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  531. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  532. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  533. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  534. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  535. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  536. package/src/inference/pipelines/text/embed.d.ts +0 -69
  537. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  538. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  539. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  540. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  541. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  542. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  543. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  544. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  545. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  546. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  547. package/src/inference/pipelines/text/ffn.js +0 -18
  548. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  549. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  550. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  551. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  552. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  553. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  554. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  555. package/src/inference/pipelines/text/index.d.ts +0 -5
  556. package/src/inference/pipelines/text/index.js +0 -6
  557. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  558. package/src/inference/pipelines/text/layer.d.ts +0 -118
  559. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  560. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  561. package/src/inference/pipelines/text/logits/types.js +0 -4
  562. package/src/inference/pipelines/text/logits.d.ts +0 -27
  563. package/src/inference/pipelines/text/logits.js +0 -16
  564. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  565. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  566. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  567. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  568. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  569. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  570. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  571. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  572. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  573. package/src/inference/pipelines/text/ops.d.ts +0 -222
  574. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  575. package/src/inference/pipelines/text/probes.d.ts +0 -28
  576. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  577. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  578. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  579. package/src/inference/pipelines/text/types.js +0 -4
  580. package/src/inference/tokenizers/base.d.ts +0 -44
  581. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  582. package/src/inference/tokenizers/bpe.d.ts +0 -27
  583. package/src/inference/tokenizers/bundled.d.ts +0 -67
  584. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  585. package/src/inference/tokenizers/types.js +0 -7
  586. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  587. package/src/loader/per-layer-input-loader.d.ts +0 -30
  588. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  589. package/src/loader/tensors/source-transform.d.ts +0 -65
  590. package/src/memory/address-table.d.ts +0 -96
  591. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  592. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  593. package/src/rules/rule-registry.d.ts +0 -77
  594. package/src/storage/artifact-storage-context.d.ts +0 -63
  595. package/src/storage/backends/idb-store.d.ts +0 -52
  596. package/src/storage/backends/memory-store.d.ts +0 -36
  597. package/src/storage/backends/opfs-store.d.ts +0 -41
  598. package/src/storage/blake3.d.ts +0 -17
  599. package/src/storage/download-types.d.ts +0 -157
  600. package/src/storage/downloader.d.ts +0 -103
  601. package/src/storage/index.d.ts +0 -274
  602. package/src/storage/index.js +0 -201
  603. package/src/storage/preflight.d.ts +0 -144
  604. package/src/storage/preflight.js +0 -360
  605. package/src/storage/quickstart-downloader.d.ts +0 -157
  606. package/src/storage/quickstart-downloader.js +0 -289
  607. package/src/storage/source-artifact-store.d.ts +0 -59
  608. package/src/sw.js +0 -184
  609. package/src/tooling/command-api-constants.d.ts +0 -9
  610. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  611. package/src/tooling/command-api-helpers.d.ts +0 -23
  612. package/src/tooling/command-runner-shared.d.ts +0 -89
  613. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  614. package/src/tooling/conversion-config-materializer.js +0 -96
  615. package/src/tooling/diagnose-runner.d.ts +0 -28
  616. package/src/tooling/hf-registry-utils.d.ts +0 -55
  617. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  618. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  619. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  620. package/src/tooling/kernel-path-builder/index.js +0 -1361
  621. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  622. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  623. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  624. package/src/tooling/lean-execution-contract-runner.js +0 -158
  625. package/src/tooling/lean-execution-contract.d.ts +0 -16
  626. package/src/tooling/lean-execution-contract.js +0 -81
  627. package/src/tooling/litert-package-runtime.d.ts +0 -46
  628. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  629. package/src/tooling/node-convert-worker.d.ts +0 -1
  630. package/src/tooling/node-converter.d.ts +0 -1
  631. package/src/tooling/node-file-fetch.d.ts +0 -1
  632. package/src/tooling/node-source-runtime.d.ts +0 -24
  633. package/src/tooling/node-webgpu.d.ts +0 -17
  634. package/src/tooling/precision-replay-math.js +0 -174
  635. package/src/tooling/program-bundle-host.d.ts +0 -16
  636. package/src/tooling/program-bundle-host.js +0 -20
  637. package/src/tooling/runtime-input-composition.d.ts +0 -37
  638. package/src/tooling/source-package-profiles.d.ts +0 -50
  639. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  640. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  641. package/src/tooling/source-runtime-materializer.js +0 -89
  642. package/src/tooling-exports.browser.d.ts +0 -7
  643. package/src/types/chrome.d.ts +0 -36
  644. package/src/types/chrome.js +0 -1
  645. package/src/types/index.d.ts +0 -1
  646. package/src/types/index.js +0 -1
  647. package/src/utils/array-buffer.d.ts +0 -4
  648. package/src/utils/canonical-hash.d.ts +0 -7
  649. package/src/utils/clone-json.d.ts +0 -1
  650. package/src/utils/hf-resolve-url.d.ts +0 -16
  651. package/src/utils/index.d.ts +0 -10
  652. package/src/utils/index.js +0 -10
  653. package/src/utils/load-json.d.ts +0 -5
  654. package/src/utils/plain-object.d.ts +0 -1
  655. package/src/utils/sha256.d.ts +0 -5
  656. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -9,7 +9,7 @@
9
9
 
10
10
  import type { CommandRecorder, ProfileTimings } from '../../../gpu/command-recorder.js';
11
11
  import type { PipelineState } from './state.js';
12
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext } from './types.js';
12
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerContext, WorkloadPhaseTiming } from './types.js';
13
13
  import type { LogitsConfig, LogitsWeights } from './logits/index.js';
14
14
  import type { WeightBufferConfig } from './weights.js';
15
15
  import type { ChatMessage } from './chat-format.js';
@@ -75,6 +75,23 @@ export declare class PipelineGenerator {
75
75
  ): Promise<DiffusionGemmaCanvasStepResult>;
76
76
  prefillWithEmbedding(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillEmbeddingResult>;
77
77
  prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
78
+ prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
79
+ seqLen: number;
80
+ tokens: number[];
81
+ tokenIds: number[];
82
+ logits: Float32Array;
83
+ logitsByTokenId: Record<number, number>;
84
+ phase?: WorkloadPhaseTiming | null;
85
+ }>;
86
+ prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
87
+ seqLen: number;
88
+ prefixTokens: number[];
89
+ tokens: number[];
90
+ tokenIds: number[];
91
+ logits: Float32Array;
92
+ logitsByTokenId: Record<number, number>;
93
+ phase?: WorkloadPhaseTiming | null;
94
+ }>;
78
95
  decodeStepLogits(currentIds: number[], options?: GenerateOptions): Promise<LogitsStepResult>;
79
96
  advanceWithToken(tokenId: number, options?: GenerateOptions): Promise<void>;
80
97
  advanceWithTokenAndEmbedding(tokenId: number, options?: GenerateOptions): Promise<AdvanceEmbeddingResult>;
@@ -134,6 +134,36 @@ const FINITENESS_RESET_WORDS = new Uint32Array(4);
134
134
  const tokenizerSuppressionCache = new WeakMap();
135
135
  const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
136
136
 
137
+ function mergeRecordOpLabelCounts(target, source) {
138
+ const merged = target && typeof target === 'object' && !Array.isArray(target)
139
+ ? target
140
+ : {};
141
+ if (!source || typeof source !== 'object' || Array.isArray(source)) {
142
+ return merged;
143
+ }
144
+ for (const [label, rawCount] of Object.entries(source)) {
145
+ const count = Number(rawCount);
146
+ if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
147
+ continue;
148
+ }
149
+ merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
150
+ }
151
+ return merged;
152
+ }
153
+
154
+ function recordPrefillRecorderStats(state, recorder) {
155
+ if (!recorder || typeof recorder.getStats !== 'function') {
156
+ return;
157
+ }
158
+ const recordStats = recorder.getStats();
159
+ state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
160
+ state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
161
+ state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
162
+ state.stats.prefillRecordOpLabels,
163
+ recordStats.opLabelCounts
164
+ );
165
+ }
166
+
137
167
  export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
138
168
  const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
139
169
  const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
@@ -219,6 +249,26 @@ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
219
249
  return [...suppressed];
220
250
  }
221
251
 
252
+ function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
253
+ if (value == null) {
254
+ return null;
255
+ }
256
+ if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
257
+ throw new Error(`${label} must be an array or typed array.`);
258
+ }
259
+ const tokenIds = Array.from(value, (entry, index) => {
260
+ const tokenId = Number(entry);
261
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
262
+ throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
263
+ }
264
+ return tokenId;
265
+ });
266
+ if (tokenIds.length === 0) {
267
+ throw new Error(`${label} must not be empty.`);
268
+ }
269
+ return tokenIds;
270
+ }
271
+
222
272
  async function traceActivationHealth(label, buffer, dtype, elementCount) {
223
273
  if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
224
274
  return;
@@ -535,6 +585,11 @@ export class PipelineGenerator {
535
585
  this.#state.stats.decodeProfileSteps = [];
536
586
  this.#state.stats.ttftMs = 0;
537
587
  this.#state.stats.decodeTimeMs = 0;
588
+ this.#state.stats.prefillRecordMs = 0;
589
+ this.#state.stats.prefillRecordOps = 0;
590
+ this.#state.stats.prefillRecordPasses = 0;
591
+ this.#state.stats.prefillRecordOpLabels = {};
592
+ this.#state.stats.prefillSubmitWaitMs = 0;
538
593
  this.#state.stats.decodeRecordMs = 0;
539
594
  this.#state.stats.decodeRecordOps = 0;
540
595
  this.#state.stats.decodeRecordPasses = 0;
@@ -667,6 +722,7 @@ export class PipelineGenerator {
667
722
 
668
723
  try {
669
724
  if (currentRecorder) {
725
+ recordPrefillRecorderStats(this.#state, currentRecorder);
670
726
  await currentRecorder.submitAndWait();
671
727
  await recordProfile(currentRecorder);
672
728
  } else {
@@ -703,12 +759,15 @@ export class PipelineGenerator {
703
759
 
704
760
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
705
761
  const prefillStartSeqLen = this.#state.currentSeqLen;
762
+ const inputStart = performance.now();
706
763
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
764
+ const inputMs = performance.now() - inputStart;
707
765
  if (opts.debug) {
708
766
  log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
709
767
  }
710
768
 
711
769
  let logits;
770
+ const prefillStart = performance.now();
712
771
  const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
713
772
  try {
714
773
  logits = await runPrefill();
@@ -726,7 +785,21 @@ export class PipelineGenerator {
726
785
  );
727
786
  }
728
787
 
729
- return { inputIds, logits };
788
+ return {
789
+ inputIds,
790
+ logits,
791
+ phase: {
792
+ inputMs,
793
+ prefillMs: performance.now() - prefillStart,
794
+ tokens: inputIds.length,
795
+ prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
796
+ prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
797
+ prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
798
+ prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
799
+ prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
800
+ gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
801
+ },
802
+ };
730
803
  }
731
804
 
732
805
  async _decodeStepToLogits(currentIds, opts) {
@@ -785,6 +858,9 @@ export class PipelineGenerator {
785
858
  this.#state.stats.decodeReadbackCleanupMs = 0;
786
859
  this.#state.stats.decodeReadbackCopyMs = 0;
787
860
  this.#state.stats.prefillRecordMs = 0;
861
+ this.#state.stats.prefillRecordOps = 0;
862
+ this.#state.stats.prefillRecordPasses = 0;
863
+ this.#state.stats.prefillRecordOpLabels = {};
788
864
  this.#state.stats.prefillSubmitWaitMs = 0;
789
865
  this.#state.stats.singleTokenSubmitWaitMs = 0;
790
866
  this.#state.stats.singleTokenReadbackWaitMs = 0;
@@ -1591,6 +1667,7 @@ export class PipelineGenerator {
1591
1667
  } = prefillResult;
1592
1668
  currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1593
1669
  if (currentRecorder) {
1670
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1594
1671
  await currentRecorder.submitAndWait();
1595
1672
  await recordProfile(currentRecorder);
1596
1673
  } else {
@@ -1678,6 +1755,7 @@ export class PipelineGenerator {
1678
1755
  } = prefillResult;
1679
1756
  currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1680
1757
  if (currentRecorder) {
1758
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1681
1759
  await currentRecorder.submitAndWait();
1682
1760
  await recordProfile(currentRecorder);
1683
1761
  } else {
@@ -1810,6 +1888,9 @@ export class PipelineGenerator {
1810
1888
  this.#state.stats.decodeReadbackCleanupMs = 0;
1811
1889
  this.#state.stats.decodeReadbackCopyMs = 0;
1812
1890
  this.#state.stats.prefillRecordMs = 0;
1891
+ this.#state.stats.prefillRecordOps = 0;
1892
+ this.#state.stats.prefillRecordPasses = 0;
1893
+ this.#state.stats.prefillRecordOpLabels = {};
1813
1894
  this.#state.stats.prefillSubmitWaitMs = 0;
1814
1895
  this.#state.stats.singleTokenSubmitWaitMs = 0;
1815
1896
  this.#state.stats.singleTokenReadbackWaitMs = 0;
@@ -1991,6 +2072,7 @@ export class PipelineGenerator {
1991
2072
 
1992
2073
  // Ensure prefill work completes before returning a usable snapshot.
1993
2074
  if (currentRecorder) {
2075
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1994
2076
  await currentRecorder.submitAndWait();
1995
2077
  await recordProfile(currentRecorder);
1996
2078
  } else {
@@ -2025,20 +2107,27 @@ export class PipelineGenerator {
2025
2107
  throw new Error('Generation already in progress');
2026
2108
  }
2027
2109
  assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
2110
+ const totalStart = performance.now();
2028
2111
  this._resetDecodeRuntimeState();
2029
2112
  this.#state.stats.gpuTimePrefillMs = undefined;
2030
2113
  this.#state.stats.prefillProfileSteps = [];
2031
2114
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2032
2115
  const prefillStartSeqLen = this.#state.currentSeqLen;
2116
+ const inputStart = performance.now();
2033
2117
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
2118
+ const inputMs = performance.now() - inputStart;
2034
2119
  if (opts.debug) {
2035
2120
  log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
2036
2121
  }
2037
2122
 
2038
2123
  try {
2039
2124
  let prefillResult;
2125
+ const prefillStart = performance.now();
2040
2126
  try {
2041
- prefillResult = await this._prefillToHidden(inputIds, opts);
2127
+ prefillResult = await this._prefillToHidden(inputIds, {
2128
+ ...opts,
2129
+ _embeddingOnly: options.__skipStateSnapshot === true,
2130
+ });
2042
2131
  } catch (error) {
2043
2132
  if (shouldRetryWithFinitenessFallback(error)) {
2044
2133
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
@@ -2046,13 +2135,17 @@ export class PipelineGenerator {
2046
2135
  opts,
2047
2136
  'prefillWithEmbedding',
2048
2137
  1,
2049
- () => this._prefillToHidden(inputIds, opts),
2138
+ () => this._prefillToHidden(inputIds, {
2139
+ ...opts,
2140
+ _embeddingOnly: options.__skipStateSnapshot === true,
2141
+ }),
2050
2142
  prefillStartSeqLen
2051
2143
  );
2052
2144
  } else {
2053
2145
  throw error;
2054
2146
  }
2055
2147
  }
2148
+ const prefillMs = performance.now() - prefillStart;
2056
2149
 
2057
2150
  const {
2058
2151
  numTokens,
@@ -2066,7 +2159,10 @@ export class PipelineGenerator {
2066
2159
  } = prefillResult;
2067
2160
 
2068
2161
  // Ensure prefill work completes before readback.
2162
+ let submitWaitMs = 0;
2163
+ const submitWaitStart = performance.now();
2069
2164
  if (currentRecorder) {
2165
+ recordPrefillRecorderStats(this.#state, currentRecorder);
2070
2166
  await currentRecorder.submitAndWait();
2071
2167
  await recordProfile(currentRecorder);
2072
2168
  } else {
@@ -2075,21 +2171,34 @@ export class PipelineGenerator {
2075
2171
  await device.queue.onSubmittedWorkDone();
2076
2172
  }
2077
2173
  }
2174
+ submitWaitMs = performance.now() - submitWaitStart;
2078
2175
 
2079
2176
  if (!allowReadback('pipeline.prefill.embedding')) {
2080
2177
  throw new Error('GPU readback disabled; cannot return embedding');
2081
2178
  }
2082
2179
 
2083
2180
  let embedding;
2181
+ let hiddenBytes = 0;
2182
+ let readbackMs = 0;
2183
+ let decodeHiddenMs = 0;
2184
+ let finalNormMs = 0;
2185
+ let extractMs = 0;
2084
2186
  try {
2085
2187
  const hiddenSize = config.hiddenSize;
2086
- const hiddenBytes = numTokens * hiddenSize * activationBytes;
2188
+ hiddenBytes = numTokens * hiddenSize * activationBytes;
2189
+ const readbackStart = performance.now();
2087
2190
  const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
2191
+ readbackMs = performance.now() - readbackStart;
2088
2192
  if (hiddenData.byteLength === 0) {
2089
2193
  throw new Error('GPU readback disabled; cannot return embedding');
2090
2194
  }
2195
+ const decodeStart = performance.now();
2091
2196
  const hiddenStates = decodeReadback(hiddenData, activationDtype);
2197
+ decodeHiddenMs = performance.now() - decodeStart;
2198
+ const finalNormStart = performance.now();
2092
2199
  const finalNormWeights = await this._getFinalNormWeights();
2200
+ finalNormMs = performance.now() - finalNormStart;
2201
+ const extractStart = performance.now();
2093
2202
  embedding = this._extractEmbeddingFromHidden(
2094
2203
  hiddenStates,
2095
2204
  numTokens,
@@ -2098,11 +2207,31 @@ export class PipelineGenerator {
2098
2207
  finalNormWeights,
2099
2208
  config
2100
2209
  );
2210
+ extractMs = performance.now() - extractStart;
2101
2211
  } finally {
2102
2212
  releaseBuffer(currentHiddenBuffer);
2103
2213
  }
2104
2214
 
2105
2215
  this.#state.currentSeqLen = startPos + numTokens;
2216
+ const phase = {
2217
+ totalMs: performance.now() - totalStart,
2218
+ inputMs,
2219
+ prefillMs,
2220
+ submitWaitMs,
2221
+ readbackMs,
2222
+ decodeHiddenMs,
2223
+ finalNormMs,
2224
+ extractMs,
2225
+ hiddenBytes,
2226
+ tokens: numTokens,
2227
+ activationDtype,
2228
+ prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
2229
+ prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
2230
+ prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
2231
+ prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
2232
+ prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
2233
+ gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
2234
+ };
2106
2235
 
2107
2236
  // Batch embedding skips expensive KV cache clone and linear attention clone
2108
2237
  // since the caller will reset immediately after extracting the embedding.
@@ -2113,6 +2242,7 @@ export class PipelineGenerator {
2113
2242
  tokens: inputIds,
2114
2243
  embedding,
2115
2244
  embeddingMode: opts.embeddingMode,
2245
+ phase,
2116
2246
  linearAttention: null,
2117
2247
  };
2118
2248
  }
@@ -2128,6 +2258,7 @@ export class PipelineGenerator {
2128
2258
  tokens: inputIds,
2129
2259
  embedding,
2130
2260
  embeddingMode: opts.embeddingMode,
2261
+ phase,
2131
2262
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2132
2263
  };
2133
2264
  } finally {
@@ -2141,12 +2272,34 @@ export class PipelineGenerator {
2141
2272
  throw new Error('Generation already in progress');
2142
2273
  }
2143
2274
  assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
2275
+ const totalStart = performance.now();
2144
2276
  this._resetDecodeRuntimeState();
2145
2277
  this.#state.stats.gpuTimePrefillMs = undefined;
2146
2278
  this.#state.stats.prefillProfileSteps = [];
2147
2279
  const opts = resolvePrefillOptions(this.#state, options);
2280
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
2281
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2282
+ if (diagnosticsEnabled) {
2283
+ const captureConfig = {
2284
+ ...createDefaultCaptureConfig(),
2285
+ enabled: true,
2286
+ defaultLevel: CAPTURE_LEVELS.SLICE,
2287
+ ...(options?.diagnostics?.captureConfig ?? {}),
2288
+ };
2289
+ validateCaptureConfig(captureConfig);
2290
+ this.#state.operatorDiagnostics = {
2291
+ enabled: true,
2292
+ captureConfig,
2293
+ emitter: new OperatorEventEmitter({
2294
+ modelHash: this.#state.manifest?.modelId ?? null,
2295
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2296
+ executionPlanHash: opts.executionPlan?.id ?? null,
2297
+ }),
2298
+ tsirFixture: null,
2299
+ };
2300
+ }
2148
2301
  try {
2149
- const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2302
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2150
2303
 
2151
2304
  const snapshot = this.#state.kvCache?.clone();
2152
2305
  if (!snapshot) {
@@ -2158,8 +2311,134 @@ export class PipelineGenerator {
2158
2311
  seqLen: this.#state.currentSeqLen,
2159
2312
  tokens: inputIds,
2160
2313
  logits,
2314
+ phase: {
2315
+ ...(prefillPhase ?? {}),
2316
+ totalMs: performance.now() - totalStart,
2317
+ },
2161
2318
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2162
2319
  };
2320
+ } finally {
2321
+ this._closeFinitenessFallbackWindow(opts);
2322
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2323
+ ? {
2324
+ enabled: true,
2325
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2326
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
2327
+ }
2328
+ : null;
2329
+ this.#state.operatorDiagnostics = null;
2330
+ }
2331
+ }
2332
+
2333
+ async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
2334
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
2335
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
2336
+ throw new Error('Generation already in progress');
2337
+ }
2338
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
2339
+ const totalStart = performance.now();
2340
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(
2341
+ tokenIds,
2342
+ this.#state.modelConfig.vocabSize,
2343
+ 'prefillWithTokenLogits.tokenIds'
2344
+ );
2345
+ this._resetDecodeRuntimeState();
2346
+ this.#state.stats.gpuTimePrefillMs = undefined;
2347
+ this.#state.stats.prefillProfileSteps = [];
2348
+ const opts = resolvePrefillOptions(this.#state, options);
2349
+ opts._selectedLogitTokenIds = selectedTokenIds;
2350
+ try {
2351
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
2352
+ const logitsByTokenId = {};
2353
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
2354
+ logitsByTokenId[selectedTokenIds[index]] = logits[index];
2355
+ }
2356
+ return {
2357
+ seqLen: this.#state.currentSeqLen,
2358
+ tokens: inputIds,
2359
+ tokenIds: selectedTokenIds,
2360
+ logits,
2361
+ logitsByTokenId,
2362
+ phase: {
2363
+ ...(prefillPhase ?? {}),
2364
+ totalMs: performance.now() - totalStart,
2365
+ selectedTokenCount: selectedTokenIds.length,
2366
+ },
2367
+ };
2368
+ } finally {
2369
+ this._closeFinitenessFallbackWindow(opts);
2370
+ }
2371
+ }
2372
+
2373
+ async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
2374
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
2375
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
2376
+ throw new Error('Generation already in progress');
2377
+ }
2378
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
2379
+ const totalStart = performance.now();
2380
+ if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
2381
+ throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
2382
+ }
2383
+ if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
2384
+ throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
2385
+ }
2386
+ if (!Array.isArray(prefix.tokens)) {
2387
+ throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
2388
+ }
2389
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(
2390
+ tokenIds,
2391
+ this.#state.modelConfig.vocabSize,
2392
+ 'prefillWithTokenLogitsFromKV.tokenIds'
2393
+ );
2394
+
2395
+ this.#state.kvCache = prefix.cache.clone();
2396
+ if (this.#state.useGPU && this.#state.kvCache) {
2397
+ const device = getDevice();
2398
+ if (device) {
2399
+ this.#state.kvCache.setGPUContext({ device });
2400
+ }
2401
+ }
2402
+ if (
2403
+ hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
2404
+ && prefix.linearAttention == null
2405
+ ) {
2406
+ throw new Error(
2407
+ 'Prefix snapshot is missing linear_attention recurrent state. ' +
2408
+ 'Regenerate the prefix snapshot using the current runtime.'
2409
+ );
2410
+ }
2411
+ this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
2412
+ this.#state.linearAttentionRuntime,
2413
+ prefix.linearAttention ?? null
2414
+ );
2415
+ this.#state.currentSeqLen = prefix.seqLen;
2416
+ this.#state.stats.gpuTimePrefillMs = undefined;
2417
+ this.#state.stats.prefillProfileSteps = [];
2418
+ this.#state.decodeRing?.reset();
2419
+
2420
+ const opts = resolvePrefillOptions(this.#state, options);
2421
+ opts._selectedLogitTokenIds = selectedTokenIds;
2422
+ try {
2423
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
2424
+ const logitsByTokenId = {};
2425
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
2426
+ logitsByTokenId[selectedTokenIds[index]] = logits[index];
2427
+ }
2428
+ return {
2429
+ seqLen: this.#state.currentSeqLen,
2430
+ prefixTokens: prefix.tokens,
2431
+ tokens: inputIds,
2432
+ tokenIds: selectedTokenIds,
2433
+ logits,
2434
+ logitsByTokenId,
2435
+ phase: {
2436
+ ...(prefillPhase ?? {}),
2437
+ totalMs: performance.now() - totalStart,
2438
+ selectedTokenCount: selectedTokenIds.length,
2439
+ prefixTokens: prefix.tokens.length,
2440
+ },
2441
+ };
2163
2442
  } finally {
2164
2443
  this._closeFinitenessFallbackWindow(opts);
2165
2444
  }
@@ -2212,6 +2491,11 @@ export class PipelineGenerator {
2212
2491
  this.#state.stats.decodeReadbackMapWaitMs = 0;
2213
2492
  this.#state.stats.decodeReadbackCleanupMs = 0;
2214
2493
  this.#state.stats.decodeReadbackCopyMs = 0;
2494
+ this.#state.stats.prefillRecordMs = 0;
2495
+ this.#state.stats.prefillRecordOps = 0;
2496
+ this.#state.stats.prefillRecordPasses = 0;
2497
+ this.#state.stats.prefillRecordOpLabels = {};
2498
+ this.#state.stats.prefillSubmitWaitMs = 0;
2215
2499
  this.#state.stats.ttftMs = 0;
2216
2500
  const startTime = performance.now();
2217
2501
 
@@ -2412,6 +2696,7 @@ export class PipelineGenerator {
2412
2696
  maxDecodeTokens: opts.maxTokens,
2413
2697
  numLayers: this.#state.modelConfig.numLayers,
2414
2698
  hiddenSize: this.#state.modelConfig.hiddenSize,
2699
+ configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
2415
2700
  });
2416
2701
  const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
2417
2702
  const boundedBatchTokens = maxBatchDecodeTokens == null
@@ -2725,6 +3010,11 @@ export class PipelineGenerator {
2725
3010
  async _prefill(inputIds, opts) {
2726
3011
  const numTokens = inputIds.length;
2727
3012
  const config = this.#state.modelConfig;
3013
+ const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
3014
+ opts?._selectedLogitTokenIds,
3015
+ config.vocabSize,
3016
+ '_prefill.selectedLogitTokenIds'
3017
+ );
2728
3018
  const startPos = this.#state.currentSeqLen;
2729
3019
  const tracePrefillEnabled = isTraceEnabled('perf');
2730
3020
  const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
@@ -2787,7 +3077,7 @@ export class PipelineGenerator {
2787
3077
  if (!device || disableCommandBatching) return undefined;
2788
3078
  return opts.profile
2789
3079
  ? createProfilingRecorder(label, device)
2790
- : createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
3080
+ : createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
2791
3081
  };
2792
3082
  const recorder = createRecorder('prefill');
2793
3083
  const debugCheckBuffer = this.#state.debug
@@ -2804,6 +3094,9 @@ export class PipelineGenerator {
2804
3094
  );
2805
3095
  context.currentTokenIds = inputIds;
2806
3096
  context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
3097
+ context.skipKVCacheWrites = returnHidden
3098
+ && opts?._embeddingOnly === true
3099
+ && this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
2807
3100
  context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
2808
3101
  ? null
2809
3102
  : {
@@ -2938,6 +3231,7 @@ export class PipelineGenerator {
2938
3231
  hiddenStates.shape,
2939
3232
  hiddenStates.label
2940
3233
  );
3234
+ recordPrefillRecorderStats(this.#state, recorder);
2941
3235
  await recorder.submitAndWait();
2942
3236
  await recordProfile(recorder);
2943
3237
  }
@@ -2972,6 +3266,7 @@ export class PipelineGenerator {
2972
3266
  // buffers, preventing unbounded memory growth during large prefills. Critical for
2973
3267
  // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2974
3268
  const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
3269
+ configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
2975
3270
  hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
2976
3271
  numTokens,
2977
3272
  });
@@ -3028,6 +3323,7 @@ export class PipelineGenerator {
3028
3323
  currentRecorder,
3029
3324
  'prefill_checkpoint_carry'
3030
3325
  );
3326
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3031
3327
  await currentRecorder.submitAndWait();
3032
3328
  await recordProfile(currentRecorder);
3033
3329
  currentRecorder = undefined;
@@ -3040,6 +3336,7 @@ export class PipelineGenerator {
3040
3336
  'prefill_trace_layer_health_carry'
3041
3337
  );
3042
3338
  const traceSubmitStart = performance.now();
3339
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3043
3340
  await currentRecorder.submitAndWait();
3044
3341
  await recordProfile(currentRecorder);
3045
3342
  prefillSubmitWaitMs += performance.now() - traceSubmitStart;
@@ -3113,6 +3410,7 @@ export class PipelineGenerator {
3113
3410
  this.#state.modelConfig
3114
3411
  );
3115
3412
  const chunkSubmitStart = performance.now();
3413
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3116
3414
  if (chunkSubmitMode === 'async' && !opts.profile) {
3117
3415
  currentRecorder.submit({ cleanup: 'deferred' });
3118
3416
  } else {
@@ -3171,6 +3469,7 @@ export class PipelineGenerator {
3171
3469
  currentRecorder,
3172
3470
  'prefill_tsir_drain_carry'
3173
3471
  );
3472
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3174
3473
  await currentRecorder.submitAndWait();
3175
3474
  await recordProfile(currentRecorder);
3176
3475
  currentRecorder = undefined;
@@ -3181,13 +3480,14 @@ export class PipelineGenerator {
3181
3480
  if (this.#state.finitenessBuffer) {
3182
3481
  if (currentRecorder) {
3183
3482
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3184
- currentHiddenBuffer,
3185
- currentRecorder,
3186
- 'prefill_finiteness_carry'
3187
- );
3188
- await currentRecorder.submitAndWait();
3189
- await recordProfile(currentRecorder);
3190
- currentRecorder = undefined;
3483
+ currentHiddenBuffer,
3484
+ currentRecorder,
3485
+ 'prefill_finiteness_carry'
3486
+ );
3487
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3488
+ await currentRecorder.submitAndWait();
3489
+ await recordProfile(currentRecorder);
3490
+ currentRecorder = undefined;
3191
3491
  }
3192
3492
  const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
3193
3493
  const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
@@ -3251,6 +3551,7 @@ export class PipelineGenerator {
3251
3551
  let usedRecordedLogits = false;
3252
3552
  const lmHead = this.#state.weights.get('lm_head');
3253
3553
  const canRecordLogits = !!currentRecorder
3554
+ && selectedLogitTokenIds === null
3254
3555
  && !!lmHead
3255
3556
  && !isCpuWeightBuffer(lmHead)
3256
3557
  && !this.#state.disableRecordedLogits
@@ -3268,6 +3569,7 @@ export class PipelineGenerator {
3268
3569
  logitsVocabSize = recorded.vocabSize;
3269
3570
  usedRecordedLogits = true;
3270
3571
 
3572
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3271
3573
  await currentRecorder.submitAndWait();
3272
3574
  await recordProfile(currentRecorder);
3273
3575
 
@@ -3329,6 +3631,7 @@ export class PipelineGenerator {
3329
3631
  currentRecorder,
3330
3632
  'prefill_logits_carry'
3331
3633
  );
3634
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3332
3635
  await currentRecorder.submitAndWait();
3333
3636
  await recordProfile(currentRecorder);
3334
3637
  }
@@ -3342,13 +3645,17 @@ export class PipelineGenerator {
3342
3645
  undefined,
3343
3646
  debugCheckBuffer,
3344
3647
  this.#state.runtimeConfig.shared.debug.probes,
3345
- { lastPositionOnly: true },
3648
+ { lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
3346
3649
  this.#state.operatorDiagnostics
3347
3650
  );
3348
3651
 
3349
- lastLogits = logits.length === logitsVocabSize
3350
- ? logits
3351
- : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
3652
+ if (selectedLogitTokenIds !== null) {
3653
+ lastLogits = logits;
3654
+ } else {
3655
+ lastLogits = logits.length === logitsVocabSize
3656
+ ? logits
3657
+ : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
3658
+ }
3352
3659
  releaseBuffer(currentHiddenBuffer);
3353
3660
  if (tracePrefillEnabled) {
3354
3661
  trace.perf('Prefill logits complete', {
@@ -3373,11 +3680,11 @@ export class PipelineGenerator {
3373
3680
  }
3374
3681
  }
3375
3682
 
3376
- if (opts.debug) {
3683
+ if (opts.debug && selectedLogitTokenIds === null) {
3377
3684
  logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
3378
3685
  }
3379
3686
  if (isTraceEnabled('logits')) {
3380
- trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3687
+ trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3381
3688
  }
3382
3689
 
3383
3690
  if (opts.debug) {