doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -134,6 +134,36 @@ const FINITENESS_RESET_WORDS = new Uint32Array(4);
134
134
  const tokenizerSuppressionCache = new WeakMap();
135
135
  const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
136
136
 
137
+ function mergeRecordOpLabelCounts(target, source) {
138
+ const merged = target && typeof target === 'object' && !Array.isArray(target)
139
+ ? target
140
+ : {};
141
+ if (!source || typeof source !== 'object' || Array.isArray(source)) {
142
+ return merged;
143
+ }
144
+ for (const [label, rawCount] of Object.entries(source)) {
145
+ const count = Number(rawCount);
146
+ if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
147
+ continue;
148
+ }
149
+ merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
150
+ }
151
+ return merged;
152
+ }
153
+
154
+ function recordPrefillRecorderStats(state, recorder) {
155
+ if (!recorder || typeof recorder.getStats !== 'function') {
156
+ return;
157
+ }
158
+ const recordStats = recorder.getStats();
159
+ state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
160
+ state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
161
+ state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
162
+ state.stats.prefillRecordOpLabels,
163
+ recordStats.opLabelCounts
164
+ );
165
+ }
166
+
137
167
  export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
138
168
  const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
139
169
  const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
@@ -219,6 +249,26 @@ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
219
249
  return [...suppressed];
220
250
  }
221
251
 
252
+ function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
253
+ if (value == null) {
254
+ return null;
255
+ }
256
+ if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
257
+ throw new Error(`${label} must be an array or typed array.`);
258
+ }
259
+ const tokenIds = Array.from(value, (entry, index) => {
260
+ const tokenId = Number(entry);
261
+ if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
262
+ throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
263
+ }
264
+ return tokenId;
265
+ });
266
+ if (tokenIds.length === 0) {
267
+ throw new Error(`${label} must not be empty.`);
268
+ }
269
+ return tokenIds;
270
+ }
271
+
222
272
  async function traceActivationHealth(label, buffer, dtype, elementCount) {
223
273
  if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
224
274
  return;
@@ -535,6 +585,11 @@ export class PipelineGenerator {
535
585
  this.#state.stats.decodeProfileSteps = [];
536
586
  this.#state.stats.ttftMs = 0;
537
587
  this.#state.stats.decodeTimeMs = 0;
588
+ this.#state.stats.prefillRecordMs = 0;
589
+ this.#state.stats.prefillRecordOps = 0;
590
+ this.#state.stats.prefillRecordPasses = 0;
591
+ this.#state.stats.prefillRecordOpLabels = {};
592
+ this.#state.stats.prefillSubmitWaitMs = 0;
538
593
  this.#state.stats.decodeRecordMs = 0;
539
594
  this.#state.stats.decodeRecordOps = 0;
540
595
  this.#state.stats.decodeRecordPasses = 0;
@@ -667,6 +722,7 @@ export class PipelineGenerator {
667
722
 
668
723
  try {
669
724
  if (currentRecorder) {
725
+ recordPrefillRecorderStats(this.#state, currentRecorder);
670
726
  await currentRecorder.submitAndWait();
671
727
  await recordProfile(currentRecorder);
672
728
  } else {
@@ -703,12 +759,15 @@ export class PipelineGenerator {
703
759
 
704
760
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
705
761
  const prefillStartSeqLen = this.#state.currentSeqLen;
762
+ const inputStart = performance.now();
706
763
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
764
+ const inputMs = performance.now() - inputStart;
707
765
  if (opts.debug) {
708
766
  log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
709
767
  }
710
768
 
711
769
  let logits;
770
+ const prefillStart = performance.now();
712
771
  const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
713
772
  try {
714
773
  logits = await runPrefill();
@@ -726,7 +785,21 @@ export class PipelineGenerator {
726
785
  );
727
786
  }
728
787
 
729
- return { inputIds, logits };
788
+ return {
789
+ inputIds,
790
+ logits,
791
+ phase: {
792
+ inputMs,
793
+ prefillMs: performance.now() - prefillStart,
794
+ tokens: inputIds.length,
795
+ prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
796
+ prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
797
+ prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
798
+ prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
799
+ prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
800
+ gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
801
+ },
802
+ };
730
803
  }
731
804
 
732
805
  async _decodeStepToLogits(currentIds, opts) {
@@ -785,6 +858,9 @@ export class PipelineGenerator {
785
858
  this.#state.stats.decodeReadbackCleanupMs = 0;
786
859
  this.#state.stats.decodeReadbackCopyMs = 0;
787
860
  this.#state.stats.prefillRecordMs = 0;
861
+ this.#state.stats.prefillRecordOps = 0;
862
+ this.#state.stats.prefillRecordPasses = 0;
863
+ this.#state.stats.prefillRecordOpLabels = {};
788
864
  this.#state.stats.prefillSubmitWaitMs = 0;
789
865
  this.#state.stats.singleTokenSubmitWaitMs = 0;
790
866
  this.#state.stats.singleTokenReadbackWaitMs = 0;
@@ -1591,6 +1667,7 @@ export class PipelineGenerator {
1591
1667
  } = prefillResult;
1592
1668
  currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1593
1669
  if (currentRecorder) {
1670
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1594
1671
  await currentRecorder.submitAndWait();
1595
1672
  await recordProfile(currentRecorder);
1596
1673
  } else {
@@ -1678,6 +1755,7 @@ export class PipelineGenerator {
1678
1755
  } = prefillResult;
1679
1756
  currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1680
1757
  if (currentRecorder) {
1758
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1681
1759
  await currentRecorder.submitAndWait();
1682
1760
  await recordProfile(currentRecorder);
1683
1761
  } else {
@@ -1810,6 +1888,9 @@ export class PipelineGenerator {
1810
1888
  this.#state.stats.decodeReadbackCleanupMs = 0;
1811
1889
  this.#state.stats.decodeReadbackCopyMs = 0;
1812
1890
  this.#state.stats.prefillRecordMs = 0;
1891
+ this.#state.stats.prefillRecordOps = 0;
1892
+ this.#state.stats.prefillRecordPasses = 0;
1893
+ this.#state.stats.prefillRecordOpLabels = {};
1813
1894
  this.#state.stats.prefillSubmitWaitMs = 0;
1814
1895
  this.#state.stats.singleTokenSubmitWaitMs = 0;
1815
1896
  this.#state.stats.singleTokenReadbackWaitMs = 0;
@@ -1991,6 +2072,7 @@ export class PipelineGenerator {
1991
2072
 
1992
2073
  // Ensure prefill work completes before returning a usable snapshot.
1993
2074
  if (currentRecorder) {
2075
+ recordPrefillRecorderStats(this.#state, currentRecorder);
1994
2076
  await currentRecorder.submitAndWait();
1995
2077
  await recordProfile(currentRecorder);
1996
2078
  } else {
@@ -2025,20 +2107,27 @@ export class PipelineGenerator {
2025
2107
  throw new Error('Generation already in progress');
2026
2108
  }
2027
2109
  assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
2110
+ const totalStart = performance.now();
2028
2111
  this._resetDecodeRuntimeState();
2029
2112
  this.#state.stats.gpuTimePrefillMs = undefined;
2030
2113
  this.#state.stats.prefillProfileSteps = [];
2031
2114
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2032
2115
  const prefillStartSeqLen = this.#state.currentSeqLen;
2116
+ const inputStart = performance.now();
2033
2117
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
2118
+ const inputMs = performance.now() - inputStart;
2034
2119
  if (opts.debug) {
2035
2120
  log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
2036
2121
  }
2037
2122
 
2038
2123
  try {
2039
2124
  let prefillResult;
2125
+ const prefillStart = performance.now();
2040
2126
  try {
2041
- prefillResult = await this._prefillToHidden(inputIds, opts);
2127
+ prefillResult = await this._prefillToHidden(inputIds, {
2128
+ ...opts,
2129
+ _embeddingOnly: options.__skipStateSnapshot === true,
2130
+ });
2042
2131
  } catch (error) {
2043
2132
  if (shouldRetryWithFinitenessFallback(error)) {
2044
2133
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
@@ -2046,13 +2135,17 @@ export class PipelineGenerator {
2046
2135
  opts,
2047
2136
  'prefillWithEmbedding',
2048
2137
  1,
2049
- () => this._prefillToHidden(inputIds, opts),
2138
+ () => this._prefillToHidden(inputIds, {
2139
+ ...opts,
2140
+ _embeddingOnly: options.__skipStateSnapshot === true,
2141
+ }),
2050
2142
  prefillStartSeqLen
2051
2143
  );
2052
2144
  } else {
2053
2145
  throw error;
2054
2146
  }
2055
2147
  }
2148
+ const prefillMs = performance.now() - prefillStart;
2056
2149
 
2057
2150
  const {
2058
2151
  numTokens,
@@ -2066,7 +2159,10 @@ export class PipelineGenerator {
2066
2159
  } = prefillResult;
2067
2160
 
2068
2161
  // Ensure prefill work completes before readback.
2162
+ let submitWaitMs = 0;
2163
+ const submitWaitStart = performance.now();
2069
2164
  if (currentRecorder) {
2165
+ recordPrefillRecorderStats(this.#state, currentRecorder);
2070
2166
  await currentRecorder.submitAndWait();
2071
2167
  await recordProfile(currentRecorder);
2072
2168
  } else {
@@ -2075,21 +2171,34 @@ export class PipelineGenerator {
2075
2171
  await device.queue.onSubmittedWorkDone();
2076
2172
  }
2077
2173
  }
2174
+ submitWaitMs = performance.now() - submitWaitStart;
2078
2175
 
2079
2176
  if (!allowReadback('pipeline.prefill.embedding')) {
2080
2177
  throw new Error('GPU readback disabled; cannot return embedding');
2081
2178
  }
2082
2179
 
2083
2180
  let embedding;
2181
+ let hiddenBytes = 0;
2182
+ let readbackMs = 0;
2183
+ let decodeHiddenMs = 0;
2184
+ let finalNormMs = 0;
2185
+ let extractMs = 0;
2084
2186
  try {
2085
2187
  const hiddenSize = config.hiddenSize;
2086
- const hiddenBytes = numTokens * hiddenSize * activationBytes;
2188
+ hiddenBytes = numTokens * hiddenSize * activationBytes;
2189
+ const readbackStart = performance.now();
2087
2190
  const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
2191
+ readbackMs = performance.now() - readbackStart;
2088
2192
  if (hiddenData.byteLength === 0) {
2089
2193
  throw new Error('GPU readback disabled; cannot return embedding');
2090
2194
  }
2195
+ const decodeStart = performance.now();
2091
2196
  const hiddenStates = decodeReadback(hiddenData, activationDtype);
2197
+ decodeHiddenMs = performance.now() - decodeStart;
2198
+ const finalNormStart = performance.now();
2092
2199
  const finalNormWeights = await this._getFinalNormWeights();
2200
+ finalNormMs = performance.now() - finalNormStart;
2201
+ const extractStart = performance.now();
2093
2202
  embedding = this._extractEmbeddingFromHidden(
2094
2203
  hiddenStates,
2095
2204
  numTokens,
@@ -2098,11 +2207,31 @@ export class PipelineGenerator {
2098
2207
  finalNormWeights,
2099
2208
  config
2100
2209
  );
2210
+ extractMs = performance.now() - extractStart;
2101
2211
  } finally {
2102
2212
  releaseBuffer(currentHiddenBuffer);
2103
2213
  }
2104
2214
 
2105
2215
  this.#state.currentSeqLen = startPos + numTokens;
2216
+ const phase = {
2217
+ totalMs: performance.now() - totalStart,
2218
+ inputMs,
2219
+ prefillMs,
2220
+ submitWaitMs,
2221
+ readbackMs,
2222
+ decodeHiddenMs,
2223
+ finalNormMs,
2224
+ extractMs,
2225
+ hiddenBytes,
2226
+ tokens: numTokens,
2227
+ activationDtype,
2228
+ prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
2229
+ prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
2230
+ prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
2231
+ prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
2232
+ prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
2233
+ gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
2234
+ };
2106
2235
 
2107
2236
  // Batch embedding skips expensive KV cache clone and linear attention clone
2108
2237
  // since the caller will reset immediately after extracting the embedding.
@@ -2113,6 +2242,7 @@ export class PipelineGenerator {
2113
2242
  tokens: inputIds,
2114
2243
  embedding,
2115
2244
  embeddingMode: opts.embeddingMode,
2245
+ phase,
2116
2246
  linearAttention: null,
2117
2247
  };
2118
2248
  }
@@ -2128,6 +2258,7 @@ export class PipelineGenerator {
2128
2258
  tokens: inputIds,
2129
2259
  embedding,
2130
2260
  embeddingMode: opts.embeddingMode,
2261
+ phase,
2131
2262
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2132
2263
  };
2133
2264
  } finally {
@@ -2141,12 +2272,13 @@ export class PipelineGenerator {
2141
2272
  throw new Error('Generation already in progress');
2142
2273
  }
2143
2274
  assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
2275
+ const totalStart = performance.now();
2144
2276
  this._resetDecodeRuntimeState();
2145
2277
  this.#state.stats.gpuTimePrefillMs = undefined;
2146
2278
  this.#state.stats.prefillProfileSteps = [];
2147
2279
  const opts = resolvePrefillOptions(this.#state, options);
2148
2280
  try {
2149
- const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2281
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2150
2282
 
2151
2283
  const snapshot = this.#state.kvCache?.clone();
2152
2284
  if (!snapshot) {
@@ -2158,6 +2290,10 @@ export class PipelineGenerator {
2158
2290
  seqLen: this.#state.currentSeqLen,
2159
2291
  tokens: inputIds,
2160
2292
  logits,
2293
+ phase: {
2294
+ ...(prefillPhase ?? {}),
2295
+ totalMs: performance.now() - totalStart,
2296
+ },
2161
2297
  linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2162
2298
  };
2163
2299
  } finally {
@@ -2165,6 +2301,120 @@ export class PipelineGenerator {
2165
2301
  }
2166
2302
  }
2167
2303
 
2304
+ async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
2305
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
2306
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
2307
+ throw new Error('Generation already in progress');
2308
+ }
2309
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
2310
+ const totalStart = performance.now();
2311
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(
2312
+ tokenIds,
2313
+ this.#state.modelConfig.vocabSize,
2314
+ 'prefillWithTokenLogits.tokenIds'
2315
+ );
2316
+ this._resetDecodeRuntimeState();
2317
+ this.#state.stats.gpuTimePrefillMs = undefined;
2318
+ this.#state.stats.prefillProfileSteps = [];
2319
+ const opts = resolvePrefillOptions(this.#state, options);
2320
+ opts._selectedLogitTokenIds = selectedTokenIds;
2321
+ try {
2322
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
2323
+ const logitsByTokenId = {};
2324
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
2325
+ logitsByTokenId[selectedTokenIds[index]] = logits[index];
2326
+ }
2327
+ return {
2328
+ seqLen: this.#state.currentSeqLen,
2329
+ tokens: inputIds,
2330
+ tokenIds: selectedTokenIds,
2331
+ logits,
2332
+ logitsByTokenId,
2333
+ phase: {
2334
+ ...(prefillPhase ?? {}),
2335
+ totalMs: performance.now() - totalStart,
2336
+ selectedTokenCount: selectedTokenIds.length,
2337
+ },
2338
+ };
2339
+ } finally {
2340
+ this._closeFinitenessFallbackWindow(opts);
2341
+ }
2342
+ }
2343
+
2344
+ async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
2345
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
2346
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
2347
+ throw new Error('Generation already in progress');
2348
+ }
2349
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
2350
+ const totalStart = performance.now();
2351
+ if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
2352
+ throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
2353
+ }
2354
+ if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
2355
+ throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
2356
+ }
2357
+ if (!Array.isArray(prefix.tokens)) {
2358
+ throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
2359
+ }
2360
+ const selectedTokenIds = normalizeSelectedLogitTokenIds(
2361
+ tokenIds,
2362
+ this.#state.modelConfig.vocabSize,
2363
+ 'prefillWithTokenLogitsFromKV.tokenIds'
2364
+ );
2365
+
2366
+ this.#state.kvCache = prefix.cache.clone();
2367
+ if (this.#state.useGPU && this.#state.kvCache) {
2368
+ const device = getDevice();
2369
+ if (device) {
2370
+ this.#state.kvCache.setGPUContext({ device });
2371
+ }
2372
+ }
2373
+ if (
2374
+ hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
2375
+ && prefix.linearAttention == null
2376
+ ) {
2377
+ throw new Error(
2378
+ 'Prefix snapshot is missing linear_attention recurrent state. ' +
2379
+ 'Regenerate the prefix snapshot using the current runtime.'
2380
+ );
2381
+ }
2382
+ this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
2383
+ this.#state.linearAttentionRuntime,
2384
+ prefix.linearAttention ?? null
2385
+ );
2386
+ this.#state.currentSeqLen = prefix.seqLen;
2387
+ this.#state.stats.gpuTimePrefillMs = undefined;
2388
+ this.#state.stats.prefillProfileSteps = [];
2389
+ this.#state.decodeRing?.reset();
2390
+
2391
+ const opts = resolvePrefillOptions(this.#state, options);
2392
+ opts._selectedLogitTokenIds = selectedTokenIds;
2393
+ try {
2394
+ const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
2395
+ const logitsByTokenId = {};
2396
+ for (let index = 0; index < selectedTokenIds.length; index += 1) {
2397
+ logitsByTokenId[selectedTokenIds[index]] = logits[index];
2398
+ }
2399
+ return {
2400
+ seqLen: this.#state.currentSeqLen,
2401
+ prefixTokens: prefix.tokens,
2402
+ tokens: inputIds,
2403
+ tokenIds: selectedTokenIds,
2404
+ logits,
2405
+ logitsByTokenId,
2406
+ phase: {
2407
+ ...(prefillPhase ?? {}),
2408
+ totalMs: performance.now() - totalStart,
2409
+ selectedTokenCount: selectedTokenIds.length,
2410
+ prefixTokens: prefix.tokens.length,
2411
+ },
2412
+ };
2413
+ } finally {
2414
+ this._closeFinitenessFallbackWindow(opts);
2415
+ }
2416
+ }
2417
+
2168
2418
 
2169
2419
  async *generateWithPrefixKV(prefix, prompt, options = {}) {
2170
2420
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
@@ -2212,6 +2462,11 @@ export class PipelineGenerator {
2212
2462
  this.#state.stats.decodeReadbackMapWaitMs = 0;
2213
2463
  this.#state.stats.decodeReadbackCleanupMs = 0;
2214
2464
  this.#state.stats.decodeReadbackCopyMs = 0;
2465
+ this.#state.stats.prefillRecordMs = 0;
2466
+ this.#state.stats.prefillRecordOps = 0;
2467
+ this.#state.stats.prefillRecordPasses = 0;
2468
+ this.#state.stats.prefillRecordOpLabels = {};
2469
+ this.#state.stats.prefillSubmitWaitMs = 0;
2215
2470
  this.#state.stats.ttftMs = 0;
2216
2471
  const startTime = performance.now();
2217
2472
 
@@ -2412,6 +2667,7 @@ export class PipelineGenerator {
2412
2667
  maxDecodeTokens: opts.maxTokens,
2413
2668
  numLayers: this.#state.modelConfig.numLayers,
2414
2669
  hiddenSize: this.#state.modelConfig.hiddenSize,
2670
+ configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
2415
2671
  });
2416
2672
  const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
2417
2673
  const boundedBatchTokens = maxBatchDecodeTokens == null
@@ -2725,6 +2981,11 @@ export class PipelineGenerator {
2725
2981
  async _prefill(inputIds, opts) {
2726
2982
  const numTokens = inputIds.length;
2727
2983
  const config = this.#state.modelConfig;
2984
+ const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
2985
+ opts?._selectedLogitTokenIds,
2986
+ config.vocabSize,
2987
+ '_prefill.selectedLogitTokenIds'
2988
+ );
2728
2989
  const startPos = this.#state.currentSeqLen;
2729
2990
  const tracePrefillEnabled = isTraceEnabled('perf');
2730
2991
  const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
@@ -2787,7 +3048,7 @@ export class PipelineGenerator {
2787
3048
  if (!device || disableCommandBatching) return undefined;
2788
3049
  return opts.profile
2789
3050
  ? createProfilingRecorder(label, device)
2790
- : createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
3051
+ : createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
2791
3052
  };
2792
3053
  const recorder = createRecorder('prefill');
2793
3054
  const debugCheckBuffer = this.#state.debug
@@ -2804,6 +3065,9 @@ export class PipelineGenerator {
2804
3065
  );
2805
3066
  context.currentTokenIds = inputIds;
2806
3067
  context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
3068
+ context.skipKVCacheWrites = returnHidden
3069
+ && opts?._embeddingOnly === true
3070
+ && this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
2807
3071
  context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
2808
3072
  ? null
2809
3073
  : {
@@ -2938,6 +3202,7 @@ export class PipelineGenerator {
2938
3202
  hiddenStates.shape,
2939
3203
  hiddenStates.label
2940
3204
  );
3205
+ recordPrefillRecorderStats(this.#state, recorder);
2941
3206
  await recorder.submitAndWait();
2942
3207
  await recordProfile(recorder);
2943
3208
  }
@@ -2972,6 +3237,7 @@ export class PipelineGenerator {
2972
3237
  // buffers, preventing unbounded memory growth during large prefills. Critical for
2973
3238
  // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2974
3239
  const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
3240
+ configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
2975
3241
  hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
2976
3242
  numTokens,
2977
3243
  });
@@ -3028,6 +3294,7 @@ export class PipelineGenerator {
3028
3294
  currentRecorder,
3029
3295
  'prefill_checkpoint_carry'
3030
3296
  );
3297
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3031
3298
  await currentRecorder.submitAndWait();
3032
3299
  await recordProfile(currentRecorder);
3033
3300
  currentRecorder = undefined;
@@ -3040,6 +3307,7 @@ export class PipelineGenerator {
3040
3307
  'prefill_trace_layer_health_carry'
3041
3308
  );
3042
3309
  const traceSubmitStart = performance.now();
3310
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3043
3311
  await currentRecorder.submitAndWait();
3044
3312
  await recordProfile(currentRecorder);
3045
3313
  prefillSubmitWaitMs += performance.now() - traceSubmitStart;
@@ -3113,6 +3381,7 @@ export class PipelineGenerator {
3113
3381
  this.#state.modelConfig
3114
3382
  );
3115
3383
  const chunkSubmitStart = performance.now();
3384
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3116
3385
  if (chunkSubmitMode === 'async' && !opts.profile) {
3117
3386
  currentRecorder.submit({ cleanup: 'deferred' });
3118
3387
  } else {
@@ -3171,6 +3440,7 @@ export class PipelineGenerator {
3171
3440
  currentRecorder,
3172
3441
  'prefill_tsir_drain_carry'
3173
3442
  );
3443
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3174
3444
  await currentRecorder.submitAndWait();
3175
3445
  await recordProfile(currentRecorder);
3176
3446
  currentRecorder = undefined;
@@ -3181,13 +3451,14 @@ export class PipelineGenerator {
3181
3451
  if (this.#state.finitenessBuffer) {
3182
3452
  if (currentRecorder) {
3183
3453
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3184
- currentHiddenBuffer,
3185
- currentRecorder,
3186
- 'prefill_finiteness_carry'
3187
- );
3188
- await currentRecorder.submitAndWait();
3189
- await recordProfile(currentRecorder);
3190
- currentRecorder = undefined;
3454
+ currentHiddenBuffer,
3455
+ currentRecorder,
3456
+ 'prefill_finiteness_carry'
3457
+ );
3458
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3459
+ await currentRecorder.submitAndWait();
3460
+ await recordProfile(currentRecorder);
3461
+ currentRecorder = undefined;
3191
3462
  }
3192
3463
  const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
3193
3464
  const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
@@ -3251,6 +3522,7 @@ export class PipelineGenerator {
3251
3522
  let usedRecordedLogits = false;
3252
3523
  const lmHead = this.#state.weights.get('lm_head');
3253
3524
  const canRecordLogits = !!currentRecorder
3525
+ && selectedLogitTokenIds === null
3254
3526
  && !!lmHead
3255
3527
  && !isCpuWeightBuffer(lmHead)
3256
3528
  && !this.#state.disableRecordedLogits
@@ -3268,6 +3540,7 @@ export class PipelineGenerator {
3268
3540
  logitsVocabSize = recorded.vocabSize;
3269
3541
  usedRecordedLogits = true;
3270
3542
 
3543
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3271
3544
  await currentRecorder.submitAndWait();
3272
3545
  await recordProfile(currentRecorder);
3273
3546
 
@@ -3329,6 +3602,7 @@ export class PipelineGenerator {
3329
3602
  currentRecorder,
3330
3603
  'prefill_logits_carry'
3331
3604
  );
3605
+ recordPrefillRecorderStats(this.#state, currentRecorder);
3332
3606
  await currentRecorder.submitAndWait();
3333
3607
  await recordProfile(currentRecorder);
3334
3608
  }
@@ -3342,13 +3616,17 @@ export class PipelineGenerator {
3342
3616
  undefined,
3343
3617
  debugCheckBuffer,
3344
3618
  this.#state.runtimeConfig.shared.debug.probes,
3345
- { lastPositionOnly: true },
3619
+ { lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
3346
3620
  this.#state.operatorDiagnostics
3347
3621
  );
3348
3622
 
3349
- lastLogits = logits.length === logitsVocabSize
3350
- ? logits
3351
- : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
3623
+ if (selectedLogitTokenIds !== null) {
3624
+ lastLogits = logits;
3625
+ } else {
3626
+ lastLogits = logits.length === logitsVocabSize
3627
+ ? logits
3628
+ : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
3629
+ }
3352
3630
  releaseBuffer(currentHiddenBuffer);
3353
3631
  if (tracePrefillEnabled) {
3354
3632
  trace.perf('Prefill logits complete', {
@@ -3373,11 +3651,11 @@ export class PipelineGenerator {
3373
3651
  }
3374
3652
  }
3375
3653
 
3376
- if (opts.debug) {
3654
+ if (opts.debug && selectedLogitTokenIds === null) {
3377
3655
  logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
3378
3656
  }
3379
3657
  if (isTraceEnabled('logits')) {
3380
- trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3658
+ trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3381
3659
  }
3382
3660
 
3383
3661
  if (opts.debug) {