doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  import { getDevice, setTrackSubmits } from '../../../gpu/device.js';
4
- import { releaseBuffer, readBuffer, readBufferSlice, uploadData } from '../../../memory/buffer-pool.js';
4
+ import { acquireBuffer, releaseBuffer, readBuffer, readBufferSlice, uploadData } from '../../../memory/buffer-pool.js';
5
5
  import { isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
6
6
  import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selection-cache.js';
7
7
  import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
@@ -14,12 +14,14 @@ import {
14
14
  validateCaptureConfig,
15
15
  } from '../../../debug/index.js';
16
16
  import { validateCallTimeOptions } from '../../../config/param-validator.js';
17
+ import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
17
18
  import { selectRuleValue } from '../../../rules/rule-registry.js';
19
+ import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
20
+ import { f32ToF16Array } from '../../kv-cache/types.js';
18
21
 
19
22
  // Pipeline sub-modules
20
23
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
21
- import { enforceLogitDrift } from '../../../hotswap/intent-bundle.js';
22
- import { applyChatTemplate, isStopToken } from './init.js';
24
+ import { applyChatTemplate, createKVCache, isStopToken } from './init.js';
23
25
  import { formatChatMessages } from './chat-format.js';
24
26
  import { embed } from './embed.js';
25
27
  import { processLayer } from './layer.js';
@@ -36,7 +38,13 @@ import {
36
38
  FinitenessError,
37
39
  advanceWithTokenAndEmbedding as runAdvanceWithTokenAndEmbedding,
38
40
  } from './generator-steps.js';
39
- import { buildLayerContext, debugCheckBuffer as debugCheckBufferHelper, getLogitsConfig, getLogitsWeights } from './generator-helpers.js';
41
+ import {
42
+ buildLayerContext,
43
+ debugCheckBuffer as debugCheckBufferHelper,
44
+ getLogitsConfig,
45
+ getLogitsWeights,
46
+ releaseSharedAttentionState,
47
+ } from './generator-helpers.js';
40
48
  import {
41
49
  assertTokenIdsInRange,
42
50
  assertTokenIdInRange,
@@ -55,8 +63,10 @@ import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
55
63
  import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
56
64
  import {
57
65
  activateFallbackExecutionPlan,
66
+ hasFallbackExecutionPlan,
58
67
  rebaseExecutionSessionPlan,
59
68
  resetActiveExecutionPlan,
69
+ resolveMaxBatchDecodeTokens,
60
70
  resolveActiveExecutionPlan,
61
71
  setActiveExecutionPlan,
62
72
  } from './execution-plan.js';
@@ -66,6 +76,26 @@ import {
66
76
  resetLinearAttentionRuntime,
67
77
  restoreLinearAttentionRuntime,
68
78
  } from './linear-attention.js';
79
+ import {
80
+ preparePerLayerInputs,
81
+ createPleBufferCache,
82
+ prefetchPerLayerRow,
83
+ ensurePleScaledProjectionNormWeight,
84
+ ensurePleGpuHotVocabularyRuntime,
85
+ ensurePleGpuSplitTablesRuntime,
86
+ getPleHotVocabularyRuntime,
87
+ hasGpuSplitPerLayerInputEmbeddings,
88
+ hasRangeBackedPerLayerInputEmbeddings,
89
+ } from './per-layer-inputs.js';
90
+ import { createTensor } from '../../../gpu/tensor.js';
91
+ import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
92
+
93
+ let intentBundleModulePromise = null;
94
+
95
+ async function getExperimentalIntentBundleModule() {
96
+ intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
97
+ return intentBundleModulePromise;
98
+ }
69
99
 
70
100
  function isStructuredChatRequest(prompt) {
71
101
  return prompt != null
@@ -74,7 +104,71 @@ function isStructuredChatRequest(prompt) {
74
104
  && Array.isArray(prompt.messages);
75
105
  }
76
106
 
107
+ export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
108
+ return hitStop !== true
109
+ && Number.isInteger(actualCount)
110
+ && Number.isInteger(requestedCount)
111
+ && actualCount > 0
112
+ && actualCount < requestedCount;
113
+ }
114
+
115
+ export function resolveHotVocabularyBatchDecodeAvailability({
116
+ hasRangeBackedPerLayerInputs,
117
+ pleHotVocabularyRuntime,
118
+ tokenId,
119
+ }) {
120
+ if (hasRangeBackedPerLayerInputs !== true) {
121
+ return false;
122
+ }
123
+ if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
124
+ return false;
125
+ }
126
+ const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
127
+ if (!Number.isInteger(sentinelIndex)) {
128
+ return false;
129
+ }
130
+ if (!Number.isInteger(tokenId)) {
131
+ return false;
132
+ }
133
+ const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
134
+ return hotTokenIndex !== sentinelIndex;
135
+ }
136
+
137
+ async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
138
+ await ensurePleGpuHotVocabularyRuntime({
139
+ config: state.modelConfig,
140
+ weights: state.weights,
141
+ perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
142
+ debugFlags: state.debugFlags,
143
+ tokenizer: state.tokenizer ?? null,
144
+ seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
145
+ });
146
+ await ensurePleGpuSplitTablesRuntime({
147
+ config: state.modelConfig,
148
+ weights: state.weights,
149
+ perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
150
+ debugFlags: state.debugFlags,
151
+ });
152
+ await ensurePleScaledProjectionNormWeight({
153
+ config: state.modelConfig,
154
+ weights: state.weights,
155
+ weightConfig: {
156
+ rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
157
+ },
158
+ debugFlags: state.debugFlags,
159
+ });
160
+ }
161
+
162
+ function recordPrefillProfileStep(state, entry) {
163
+ if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
164
+ if (!state.stats.prefillProfileSteps) {
165
+ state.stats.prefillProfileSteps = [];
166
+ }
167
+ state.stats.prefillProfileSteps.push(entry);
168
+ }
169
+
77
170
  function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
171
+ const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
78
172
  if (typeof prompt === 'string') {
79
173
  if (useChatTemplate && state.modelConfig.chatTemplateType) {
80
174
  if (state.modelConfig.chatTemplateType === 'translategemma') {
@@ -83,7 +177,7 @@ function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
83
177
  'Pass { messages: [...] } instead of a plain string prompt.'
84
178
  );
85
179
  }
86
- return applyChatTemplate(prompt, state.modelConfig.chatTemplateType);
180
+ return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
87
181
  }
88
182
  return prompt;
89
183
  }
@@ -103,7 +197,227 @@ function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
103
197
  );
104
198
  }
105
199
  const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
106
- return formatChatMessages(messages, templateType);
200
+ return formatChatMessages(messages, templateType, chatOptions);
201
+ }
202
+
203
+ function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
204
+ if (!buffer) {
205
+ return;
206
+ }
207
+ const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
208
+ if (ownsBuffer) {
209
+ return;
210
+ }
211
+ const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
212
+ if (cachedPleBuffer) {
213
+ return;
214
+ }
215
+ if (recorder) {
216
+ recorder.trackTemporaryBuffer(buffer);
217
+ return;
218
+ }
219
+ releaseBuffer(buffer);
220
+ }
221
+
222
+ function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
223
+ if (override == null) {
224
+ return null;
225
+ }
226
+ if (typeof override !== 'object') {
227
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
228
+ }
229
+
230
+ const prefixLength = Number(override.prefixLength);
231
+ if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
232
+ throw new Error(
233
+ `[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
234
+ );
235
+ }
236
+ if (prefixLength === 0) {
237
+ return null;
238
+ }
239
+
240
+ const offset = Number(override.offset ?? 0);
241
+ if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
242
+ throw new Error(
243
+ `[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
244
+ );
245
+ }
246
+ if (offset + prefixLength > numTokens) {
247
+ throw new Error(
248
+ `[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
249
+ );
250
+ }
251
+
252
+ const embeddings = override.embeddings ?? null;
253
+ if (!embeddings) {
254
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
255
+ }
256
+
257
+ const expectedLength = prefixLength * hiddenSize;
258
+ if (embeddings instanceof Float32Array) {
259
+ if (embeddings.length !== expectedLength) {
260
+ throw new Error(
261
+ `[Pipeline] ${contextLabel}: embedding override length mismatch ` +
262
+ `(expected=${expectedLength}, got=${embeddings.length}).`
263
+ );
264
+ }
265
+ } else if (isGpuBufferInstance(embeddings)) {
266
+ const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
267
+ if (embeddings.size < expectedBytes) {
268
+ throw new Error(
269
+ `[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
270
+ `(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
271
+ );
272
+ }
273
+ } else {
274
+ throw new Error(
275
+ `[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
276
+ );
277
+ }
278
+
279
+ return {
280
+ prefixLength,
281
+ offset,
282
+ embeddings,
283
+ expectedLength,
284
+ byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
285
+ byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
286
+ };
287
+ }
288
+
289
+ function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
290
+ if (embeddingInputSpan == null) {
291
+ return inputIds;
292
+ }
293
+ if (typeof embeddingInputSpan !== 'object') {
294
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
295
+ }
296
+
297
+ const offset = Number(embeddingInputSpan.offset);
298
+ const length = Number(embeddingInputSpan.length);
299
+ const tokenId = Number(embeddingInputSpan.tokenId);
300
+ if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
301
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
302
+ }
303
+ if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
304
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
305
+ }
306
+ if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
307
+ throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
308
+ }
309
+ if (offset + length > inputIds.length) {
310
+ throw new Error(
311
+ `[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
312
+ `exceeds inputIds length ${inputIds.length}.`
313
+ );
314
+ }
315
+ const replacedInputIds = Array.from(inputIds);
316
+ replacedInputIds.fill(tokenId, offset, offset + length);
317
+ return replacedInputIds;
318
+ }
319
+
320
+ function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
321
+ if (bidirectionalSpan == null) {
322
+ return null;
323
+ }
324
+ if (typeof bidirectionalSpan !== 'object') {
325
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
326
+ }
327
+
328
+ const offset = Number(bidirectionalSpan.offset);
329
+ const length = Number(bidirectionalSpan.length);
330
+ if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
331
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
332
+ }
333
+ if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
334
+ throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
335
+ }
336
+ if ((offset + length) > inputIds.length) {
337
+ throw new Error(
338
+ `[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
339
+ `exceeds inputIds length ${inputIds.length}.`
340
+ );
341
+ }
342
+ return { offset, length };
343
+ }
344
+
345
+ async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
346
+ if (!override) {
347
+ return baseTensor;
348
+ }
349
+ if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
350
+ throw new Error(
351
+ `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
352
+ );
353
+ }
354
+
355
+ const device = getDevice();
356
+ if (!device) {
357
+ throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
358
+ }
359
+
360
+ const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
361
+ const targetElementOffset = override.offset * hiddenSize;
362
+ const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
363
+ try {
364
+ const encoder = device.createCommandEncoder();
365
+ encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
366
+ if (baseTensor.dtype === 'f32') {
367
+ if (isGpuBufferInstance(override.embeddings)) {
368
+ encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
369
+ device.queue.submit([encoder.finish()]);
370
+ } else {
371
+ device.queue.submit([encoder.finish()]);
372
+ uploadData(outputBuffer, override.embeddings, override.byteOffset);
373
+ }
374
+ return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
375
+ }
376
+
377
+ if (isGpuBufferInstance(override.embeddings)) {
378
+ const overrideTensor = createTensor(
379
+ override.embeddings,
380
+ 'f32',
381
+ [override.prefixLength, hiddenSize],
382
+ 'prefill_embedding_override_f32'
383
+ );
384
+ assertImplicitDtypeTransitionAllowed({
385
+ executionPolicies,
386
+ fromDtype: 'f32',
387
+ toDtype: 'f16',
388
+ op: 'prefill_embedding_override',
389
+ detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
390
+ });
391
+ const castedOverride = await castF32ToF16(overrideTensor);
392
+ try {
393
+ encoder.copyBufferToBuffer(
394
+ castedOverride.buffer,
395
+ 0,
396
+ outputBuffer,
397
+ targetByteOffset,
398
+ castedOverride.buffer.size
399
+ );
400
+ device.queue.submit([encoder.finish()]);
401
+ } finally {
402
+ releaseBuffer(castedOverride.buffer);
403
+ }
404
+ } else {
405
+ assertImplicitDtypeTransitionAllowed({
406
+ executionPolicies,
407
+ fromDtype: 'f32',
408
+ toDtype: 'f16',
409
+ op: 'prefill_embedding_override',
410
+ detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
411
+ });
412
+ const packedOverride = f32ToF16Array(override.embeddings);
413
+ device.queue.submit([encoder.finish()]);
414
+ uploadData(outputBuffer, packedOverride, targetByteOffset);
415
+ }
416
+ return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
417
+ } catch (error) {
418
+ releaseBuffer(outputBuffer);
419
+ throw error;
420
+ }
107
421
  }
108
422
 
109
423
  /**
@@ -147,6 +461,44 @@ function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenTe
147
461
  return fallbackText;
148
462
  }
149
463
 
464
+ function usesReplayPrefillDecode(state) {
465
+ return state?.modelConfig?.decodeStrategy === 'replay_prefill';
466
+ }
467
+
468
+ export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
469
+ if (
470
+ opts?.disableCommandBatching === true
471
+ || opts?.debug === true
472
+ || (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
473
+ ) {
474
+ return true;
475
+ }
476
+ if (state?.kvCache?.layout === 'bdpa_paged') {
477
+ return true;
478
+ }
479
+ if (multimodalBidirectionalSpan == null) {
480
+ return false;
481
+ }
482
+ if (state?.kvCache?.hasGPUCache?.() !== true) {
483
+ return false;
484
+ }
485
+ // WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
486
+ // multimodal logits on the first generated token. Keep this lane fail-closed
487
+ // until the recorder/cache interaction is numerically matched to the direct path.
488
+ return true;
489
+ }
490
+
491
+ function assertIncrementalDecodeSupport(state, operation) {
492
+ if (!usesReplayPrefillDecode(state)) {
493
+ return;
494
+ }
495
+ throw new Error(
496
+ `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
497
+ 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
498
+ 'explicit layerTypes for mixed-geometry/shared-KV decode.'
499
+ );
500
+ }
501
+
150
502
  function summarizeExecutionPlan(plan) {
151
503
  if (!plan) {
152
504
  return null;
@@ -189,6 +541,64 @@ export function shouldRetryWithFinitenessFallback(error) {
189
541
  || message.includes('Softmax produced no finite candidate probabilities');
190
542
  }
191
543
 
544
+ function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
545
+ const activePlan = resolveActiveExecutionPlan(state);
546
+ const wrapped = new Error(
547
+ `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
548
+ `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
549
+ 'Resolve the unstable path with an explicit capability-aware execution override, ' +
550
+ 'or opt into alternate-plan recovery with ' +
551
+ 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
552
+ error instanceof Error ? { cause: error } : undefined
553
+ );
554
+ wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
555
+ return wrapped;
556
+ }
557
+
558
+ function assertResolvedKVDtype(kvDtype, contextLabel) {
559
+ if (kvDtype === 'f16' || kvDtype === 'f32') {
560
+ return kvDtype;
561
+ }
562
+ throw new Error(
563
+ `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
564
+ `got ${JSON.stringify(kvDtype)}.`
565
+ );
566
+ }
567
+
568
+ function resolveTargetPlanKVDtype(plan, contextLabel) {
569
+ return assertResolvedKVDtype(
570
+ plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
571
+ contextLabel
572
+ );
573
+ }
574
+
575
+ function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
576
+ return assertResolvedKVDtype(
577
+ state?.kvCache?.kvDtype
578
+ ?? plan?.kernelPath?.kvDtype
579
+ ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
580
+ ?? plan?.activationDtype
581
+ ?? null,
582
+ contextLabel
583
+ );
584
+ }
585
+
586
+ function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
587
+ const runtimeInference = state?.runtimeConfig?.inference;
588
+ if (!runtimeInference?.session?.kvcache) {
589
+ throw new Error(
590
+ '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
591
+ );
592
+ }
593
+ return mergeRuntimeValues(runtimeInference, {
594
+ session: {
595
+ kvcache: {
596
+ kvDtype,
597
+ },
598
+ },
599
+ });
600
+ }
601
+
192
602
  export class PipelineGenerator {
193
603
 
194
604
  #state;
@@ -222,7 +632,62 @@ export class PipelineGenerator {
222
632
  return this.#finitenessFallbackWindow !== null;
223
633
  }
224
634
 
225
- _openFinitenessFallbackWindow(opts, reasonLabel, tokenCount) {
635
+ _resetReplayPrefillRuntimeState() {
636
+ this.#state.kvCache?.clear?.();
637
+ this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
638
+ this.#state.currentSeqLen = 0;
639
+ }
640
+
641
+ async _replayPrefillDecodeLogits(currentIds, opts) {
642
+ // Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
643
+ // Without KV cache creation, this bound is not enforced elsewhere.
644
+ const kvConfig = this.#state.runtimeConfig?.inference?.session?.kvcache;
645
+ const replayMaxSeqLen = kvConfig?.maxSeqLen;
646
+ if (Number.isFinite(replayMaxSeqLen) && replayMaxSeqLen > 0 && currentIds.length > replayMaxSeqLen) {
647
+ throw new Error(
648
+ `[Pipeline] Replay-prefill sequence length ${currentIds.length} exceeds ` +
649
+ `runtime.inference.session.kvcache.maxSeqLen (${replayMaxSeqLen}). ` +
650
+ 'Increase maxSeqLen in a tier profile or runtime config to allow longer sequences.'
651
+ );
652
+ }
653
+ this.#state.decodeStepCount++;
654
+ this._resetReplayPrefillRuntimeState();
655
+ const logits = await this._prefill(currentIds, opts);
656
+ return {
657
+ logits,
658
+ logitsBuffer: null,
659
+ logitsDtype: null,
660
+ rawVocabSize: this.#state.modelConfig.vocabSize,
661
+ vocabSize: this.#state.modelConfig.vocabSize,
662
+ };
663
+ }
664
+
665
+ _shouldUseFinitenessFallback(error, contextLabel) {
666
+ if (!shouldRetryWithFinitenessFallback(error)) {
667
+ return false;
668
+ }
669
+ if (!hasFallbackExecutionPlan(this.#state)) {
670
+ throw createUnhandledFinitenessPolicyError(this.#state, contextLabel, error);
671
+ }
672
+ return true;
673
+ }
674
+
675
+ _recreateKVCacheForExecutionPlan(plan, reasonLabel) {
676
+ const kvDtype = resolveTargetPlanKVDtype(plan, `${reasonLabel}: target plan`);
677
+ const runtimeInference = cloneRuntimeInferenceWithKVDtype(this.#state, kvDtype);
678
+ this.#state.kvCache?.destroy?.();
679
+ this.#state.kvCache = createKVCache(
680
+ this.#state.modelConfig,
681
+ this.#state.useGPU,
682
+ this.#state.debug,
683
+ runtimeInference
684
+ );
685
+ this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
686
+ this.#state.currentSeqLen = 0;
687
+ return kvDtype;
688
+ }
689
+
690
+ _openFinitenessFallbackWindow(opts, reasonLabel, tokenCount, rollbackSeqLen = undefined) {
226
691
  const normalizedCount = Number.isFinite(tokenCount)
227
692
  ? Math.max(1, Math.floor(tokenCount))
228
693
  : 1;
@@ -233,7 +698,7 @@ export class PipelineGenerator {
233
698
  );
234
699
  return;
235
700
  }
236
- const original = this._beginFinitenessFallback(opts, reasonLabel);
701
+ const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
237
702
  this.#finitenessFallbackWindow = {
238
703
  original,
239
704
  remainingTokens: normalizedCount,
@@ -264,6 +729,7 @@ export class PipelineGenerator {
264
729
  }
265
730
 
266
731
  _resetDecodeRuntimeState() {
732
+ this.#state.stats.prefillProfileSteps = [];
267
733
  this.#state.stats.decodeMode = null;
268
734
  this.#state.stats.batchGuardReason = null;
269
735
  this.#state.stats.decodeProfileSteps = [];
@@ -292,6 +758,7 @@ export class PipelineGenerator {
292
758
  buildLayerContext(this.#state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
293
759
  getLogitsWeights: () => getLogitsWeights(this.#state),
294
760
  getLogitsConfig: () => getLogitsConfig(this.#state),
761
+ releaseSharedAttentionState,
295
762
  debugCheckBuffer,
296
763
  };
297
764
  }
@@ -319,6 +786,14 @@ export class PipelineGenerator {
319
786
  return inputIds;
320
787
  }
321
788
 
789
+ _resolvePromptOrInputIds(prompt, useChatTemplate, contextLabel, explicitInputIds = null) {
790
+ if (Array.isArray(explicitInputIds)) {
791
+ this._assertTokenIdsInRange(explicitInputIds, `${contextLabel}.inputIds`);
792
+ return explicitInputIds;
793
+ }
794
+ return this._resolvePromptTokenIds(prompt, useChatTemplate, contextLabel);
795
+ }
796
+
322
797
  _sampleNextTokenFromLogits(logits, generatedIds, opts) {
323
798
  const sampledLogits = Float32Array.from(logits);
324
799
  applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
@@ -333,7 +808,8 @@ export class PipelineGenerator {
333
808
  }
334
809
 
335
810
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
336
- const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, contextLabel);
811
+ const prefillStartSeqLen = this.#state.currentSeqLen;
812
+ const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
337
813
  if (opts.debug) {
338
814
  log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
339
815
  }
@@ -342,14 +818,16 @@ export class PipelineGenerator {
342
818
  try {
343
819
  logits = await this._prefill(inputIds, opts);
344
820
  } catch (error) {
345
- if (!shouldRetryWithFinitenessFallback(error)) {
821
+ if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
346
822
  throw error;
347
823
  }
348
824
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
349
- logits = await this._retryWithFinitenessFallback(
825
+ logits = await this._retryWithPersistentFinitenessFallback(
350
826
  opts,
351
827
  contextLabel,
352
- () => this._prefill(inputIds, opts)
828
+ opts.maxTokens ?? 1,
829
+ () => this._prefill(inputIds, opts),
830
+ prefillStartSeqLen
353
831
  );
354
832
  }
355
833
 
@@ -357,6 +835,9 @@ export class PipelineGenerator {
357
835
  }
358
836
 
359
837
  async _decodeStepToLogits(currentIds, opts) {
838
+ if (usesReplayPrefillDecode(this.#state)) {
839
+ return this._replayPrefillDecodeLogits(currentIds, opts);
840
+ }
360
841
  const debugCheckBuffer = this.#state.debug
361
842
  ? (buffer, label, numTokens, expectedDim) =>
362
843
  debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
@@ -369,6 +850,21 @@ export class PipelineGenerator {
369
850
  return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
370
851
  }
371
852
 
853
+ _matchesStopSequence(generatedIds, stopSequenceStart, stopSequences) {
854
+ if (!Array.isArray(stopSequences) || stopSequences.length === 0) {
855
+ return false;
856
+ }
857
+ const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
858
+ return stopSequences.some((sequence) => fullText.endsWith(sequence));
859
+ }
860
+
861
+ _shouldStopAfterAppendedToken(generatedIds, tokenId, opts, runtime) {
862
+ if (isStopToken(tokenId, runtime.stopTokenIds, runtime.eosToken)) {
863
+ return true;
864
+ }
865
+ return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
866
+ }
867
+
372
868
  async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
373
869
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
374
870
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
@@ -385,6 +881,12 @@ export class PipelineGenerator {
385
881
  this.#state.stats.singleTokenSubmitWaitMs = 0;
386
882
  this.#state.stats.singleTokenReadbackWaitMs = 0;
387
883
  this.#state.stats.singleTokenOrchestrationMs = 0;
884
+ this.#state.stats.pleHotVocabularyHits = 0;
885
+ this.#state.stats.pleHotVocabularyMisses = 0;
886
+ this.#state.stats.plePreparedTokenCacheHits = 0;
887
+ this.#state.stats.plePreparedTokenCacheMisses = 0;
888
+ this.#state.stats.plePreparedTokenCacheEntries = 0;
889
+ this.#state.stats.plePreparedTokenCacheBytes = 0;
388
890
  this.#state.stats.decodeMode = null;
389
891
  this.#state.stats.batchGuardReason = null;
390
892
  this.#state.stats.ttftMs = 0;
@@ -444,6 +946,7 @@ export class PipelineGenerator {
444
946
  };
445
947
 
446
948
  try {
949
+ const prefillStartSeqLen = this.#state.currentSeqLen;
447
950
  const prefillStart = performance.now();
448
951
  const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
449
952
  let prefillLogits = initialPrefillLogits;
@@ -464,6 +967,7 @@ export class PipelineGenerator {
464
967
  ?? intentBundle?.constraints?.max_drift_threshold;
465
968
 
466
969
  if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
970
+ const { enforceLogitDrift } = await getExperimentalIntentBundleModule();
467
971
  const actualTopK = getTopK(
468
972
  prefillLogits,
469
973
  expectedTopK.length,
@@ -488,14 +992,16 @@ export class PipelineGenerator {
488
992
  try {
489
993
  firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
490
994
  } catch (error) {
491
- if (!shouldRetryWithFinitenessFallback(error)) {
995
+ if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
492
996
  throw error;
493
997
  }
494
998
  log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
495
- prefillLogits = await this._retryWithFinitenessFallback(
999
+ prefillLogits = await this._retryWithPersistentFinitenessFallback(
496
1000
  opts,
497
1001
  'prefill-sample',
498
- () => this._prefill(inputIds, opts)
1002
+ opts.maxTokens,
1003
+ () => this._prefill(inputIds, opts),
1004
+ prefillStartSeqLen
499
1005
  );
500
1006
  firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
501
1007
  }
@@ -505,6 +1011,9 @@ export class PipelineGenerator {
505
1011
  log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
506
1012
  }
507
1013
 
1014
+ const stopTokenIds = this.#state.modelConfig.stopTokenIds;
1015
+ const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
1016
+ const stopSequenceStart = inputIds.length;
508
1017
  generatedIds.push(firstToken);
509
1018
  this.#state.stats.ttftMs = performance.now() - startTime;
510
1019
 
@@ -515,18 +1024,25 @@ export class PipelineGenerator {
515
1024
  (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
516
1025
  (tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
517
1026
  );
518
-
519
- yield* emitToken(this, firstToken, decodeToken);
520
-
521
- yield* this._runDecodeLoop(generatedIds, opts, options, {
522
- stopTokenIds: this.#state.modelConfig.stopTokenIds,
523
- eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
524
- stopSequenceStart: inputIds.length,
1027
+ const decodeRuntime = {
1028
+ stopTokenIds,
1029
+ eosToken,
1030
+ stopSequenceStart,
525
1031
  decodeToken,
526
1032
  logBatchPath: opts.debug,
527
1033
  emitMode: mode,
528
- });
529
- const tokensGenerated = this.#state.stats.decodeTokens;
1034
+ };
1035
+
1036
+ yield* emitToken(this, firstToken, decodeToken);
1037
+
1038
+ if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1039
+ this.#state.stats.decodeTimeMs = 0;
1040
+ this.#state.stats.tokensGenerated = 1;
1041
+ this.#state.stats.decodeTokens = 1;
1042
+ } else {
1043
+ yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
1044
+ }
1045
+ const tokensGenerated = this.#state.stats.decodeTokens ?? 1;
530
1046
  this.#state.stats.totalTimeMs = performance.now() - startTime;
531
1047
 
532
1048
  if (opts.debug) {
@@ -566,26 +1082,78 @@ export class PipelineGenerator {
566
1082
  }
567
1083
  }
568
1084
 
569
- _beginFinitenessFallback(opts, reasonLabel) {
1085
+ _beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen = undefined) {
570
1086
  const originalPlan = resolveActiveExecutionPlan(this.#state);
1087
+ const currentKvDtype = resolveCurrentKVCacheDtype(
1088
+ this.#state,
1089
+ originalPlan,
1090
+ `${reasonLabel}: current plan`
1091
+ );
571
1092
  const original = {
572
1093
  activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
573
1094
  seed: opts.seed,
1095
+ restoreKVCachePlan: null,
574
1096
  };
575
1097
 
576
1098
  const fallbackPlan = activateFallbackExecutionPlan(this.#state);
577
1099
  if (!fallbackPlan) {
578
- throw new Error('[Pipeline] Finiteness fallback plan is unavailable for this model/runtime configuration.');
1100
+ throw new Error(
1101
+ '[Pipeline] Explicit alternate-plan finiteness recovery is unavailable for this model/runtime configuration.'
1102
+ );
579
1103
  }
580
1104
  log.warn(
581
1105
  'Pipeline',
582
1106
  `FinitenessGuard fallback (${reasonLabel}): ` +
583
1107
  `${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
584
1108
  );
1109
+ const fallbackKvDtype = resolveTargetPlanKVDtype(
1110
+ fallbackPlan,
1111
+ `${reasonLabel}: fallback plan`
1112
+ );
1113
+
1114
+ if (Number.isInteger(rollbackSeqLen) && rollbackSeqLen < 0) {
1115
+ setActiveExecutionPlan(this.#state, original.activePlanId);
1116
+ throw new Error(
1117
+ `[Pipeline] ${reasonLabel}: rollbackSeqLen must be a non-negative integer when provided.`
1118
+ );
1119
+ }
1120
+
1121
+ if (fallbackKvDtype !== currentKvDtype) {
1122
+ if (rollbackSeqLen !== 0) {
1123
+ setActiveExecutionPlan(this.#state, original.activePlanId);
1124
+ throw new Error(
1125
+ `[Pipeline] ${reasonLabel}: finiteness fallback requires rebuilding the KV cache ` +
1126
+ `${currentKvDtype} -> ${fallbackKvDtype}, which is only supported from a fresh prefill (rollbackSeqLen=0).`
1127
+ );
1128
+ }
1129
+ try {
1130
+ this._recreateKVCacheForExecutionPlan(fallbackPlan, reasonLabel);
1131
+ original.restoreKVCachePlan = originalPlan;
1132
+ } catch (error) {
1133
+ setActiveExecutionPlan(this.#state, original.activePlanId);
1134
+ try {
1135
+ this._recreateKVCacheForExecutionPlan(originalPlan, `${reasonLabel}: restore primary`);
1136
+ } catch (restoreError) {
1137
+ log.warn(
1138
+ 'Pipeline',
1139
+ `Failed to restore primary KV cache after fallback activation error: ${restoreError}`
1140
+ );
1141
+ }
1142
+ throw error;
1143
+ }
1144
+ } else if (Number.isInteger(rollbackSeqLen)) {
1145
+ this.#state.kvCache?.truncate(rollbackSeqLen);
1146
+ this.#state.currentSeqLen = rollbackSeqLen;
1147
+ if (rollbackSeqLen === 0) {
1148
+ this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
1149
+ }
1150
+ } else {
1151
+ this.#state.kvCache?.truncate(this.#state.currentSeqLen);
1152
+ }
585
1153
 
586
1154
  this.#state.decodeBuffers?.ensureBuffers({
587
1155
  hiddenSize: this.#state.modelConfig.hiddenSize,
588
- intermediateSize: this.#state.modelConfig.intermediateSize,
1156
+ intermediateSize: this.#state.modelConfig.maxIntermediateSize,
589
1157
  activationDtype: fallbackPlan.activationDtype,
590
1158
  enablePingPong: true,
591
1159
  });
@@ -619,6 +1187,9 @@ export class PipelineGenerator {
619
1187
  setActiveExecutionPlan(this.#state, original.activePlanId);
620
1188
  opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
621
1189
  const restoredPlan = resolveActiveExecutionPlan(this.#state);
1190
+ if (original.restoreKVCachePlan) {
1191
+ this._recreateKVCacheForExecutionPlan(restoredPlan, 'restore-primary-plan');
1192
+ }
622
1193
  if (this.#state.stats.executionPlan) {
623
1194
  this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
624
1195
  this.#state.stats.executionPlan.transitions.push({
@@ -637,18 +1208,17 @@ export class PipelineGenerator {
637
1208
  const nextActivationDtype = this._getEffectiveActivationDtype();
638
1209
  this.#state.decodeBuffers?.ensureBuffers({
639
1210
  hiddenSize: this.#state.modelConfig.hiddenSize,
640
- intermediateSize: this.#state.modelConfig.intermediateSize,
1211
+ intermediateSize: this.#state.modelConfig.maxIntermediateSize,
641
1212
  activationDtype: nextActivationDtype,
642
1213
  enablePingPong: true,
643
1214
  });
644
1215
  }
645
1216
 
646
- async _retryWithFinitenessFallback(opts, reasonLabel, retryFn) {
1217
+ async _retryWithFinitenessFallback(opts, reasonLabel, retryFn, rollbackSeqLen = undefined) {
647
1218
  if (this._hasFinitenessFallbackWindow()) {
648
1219
  return retryFn();
649
1220
  }
650
- this.#state.kvCache?.truncate(this.#state.currentSeqLen);
651
- const original = this._beginFinitenessFallback(opts, reasonLabel);
1221
+ const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
652
1222
  try {
653
1223
  return await retryFn();
654
1224
  } finally {
@@ -656,6 +1226,19 @@ export class PipelineGenerator {
656
1226
  }
657
1227
  }
658
1228
 
1229
+ async _retryWithPersistentFinitenessFallback(opts, reasonLabel, tokenBudget, retryFn, rollbackSeqLen = undefined) {
1230
+ if (this._hasFinitenessFallbackWindow()) {
1231
+ return retryFn();
1232
+ }
1233
+ this._openFinitenessFallbackWindow(opts, reasonLabel, tokenBudget, rollbackSeqLen);
1234
+ try {
1235
+ return await retryFn();
1236
+ } catch (error) {
1237
+ this._closeFinitenessFallbackWindow(opts);
1238
+ throw error;
1239
+ }
1240
+ }
1241
+
659
1242
  async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
660
1243
  const windowTokens = this._resolveDeferredRoundingWindowTokens();
661
1244
  if (windowTokens <= 1) {
@@ -666,7 +1249,6 @@ export class PipelineGenerator {
666
1249
  );
667
1250
  }
668
1251
 
669
- this.#state.kvCache?.truncate(this.#state.currentSeqLen);
670
1252
  this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
671
1253
  try {
672
1254
  return await this._decodeStep(generatedIds, opts);
@@ -705,6 +1287,12 @@ export class PipelineGenerator {
705
1287
  this.#state.stats.singleTokenSubmitWaitMs = 0;
706
1288
  this.#state.stats.singleTokenReadbackWaitMs = 0;
707
1289
  this.#state.stats.singleTokenOrchestrationMs = 0;
1290
+ this.#state.stats.pleHotVocabularyHits = 0;
1291
+ this.#state.stats.pleHotVocabularyMisses = 0;
1292
+ this.#state.stats.plePreparedTokenCacheHits = 0;
1293
+ this.#state.stats.plePreparedTokenCacheMisses = 0;
1294
+ this.#state.stats.plePreparedTokenCacheEntries = 0;
1295
+ this.#state.stats.plePreparedTokenCacheBytes = 0;
708
1296
  this.#state.stats.decodeMode = null;
709
1297
  this.#state.stats.batchGuardReason = null;
710
1298
  this.#state.stats.ttftMs = 0;
@@ -718,6 +1306,7 @@ export class PipelineGenerator {
718
1306
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
719
1307
 
720
1308
  try {
1309
+ const prefillStartSeqLen = this.#state.currentSeqLen;
721
1310
  const prefillStart = performance.now();
722
1311
  const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
723
1312
  let prefillLogits = initialPrefillLogits;
@@ -730,58 +1319,45 @@ export class PipelineGenerator {
730
1319
  try {
731
1320
  firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
732
1321
  } catch (error) {
733
- if (!shouldRetryWithFinitenessFallback(error)) {
1322
+ if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
734
1323
  throw error;
735
1324
  }
736
- prefillLogits = await this._retryWithFinitenessFallback(
1325
+ prefillLogits = await this._retryWithPersistentFinitenessFallback(
737
1326
  opts,
738
1327
  'prefill-sample',
739
- () => this._prefill(inputIds, opts)
1328
+ opts.maxTokens,
1329
+ () => this._prefill(inputIds, opts),
1330
+ prefillStartSeqLen
740
1331
  );
741
1332
  firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
742
1333
  }
743
1334
 
744
- generatedIds.push(firstToken);
745
- const tokenIds = [firstToken];
746
- this.#state.stats.ttftMs = performance.now() - startTime;
747
-
748
1335
  const stopTokenIds = this.#state.modelConfig.stopTokenIds;
749
1336
  const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
750
1337
  const stopSequenceStart = inputIds.length;
1338
+ generatedIds.push(firstToken);
1339
+ const tokenIds = [firstToken];
1340
+ this.#state.stats.ttftMs = performance.now() - startTime;
751
1341
  markKernelCacheWarmed();
752
- const decodeStart = performance.now();
753
1342
 
754
- while (tokenIds.length < opts.maxTokens) {
755
- if (options.signal?.aborted) break;
756
- let nextToken;
757
- try {
758
- nextToken = await this._decodeNextTokenViaLogits(generatedIds, opts);
759
- } catch (error) {
760
- if (shouldRetryWithFinitenessFallback(error)) {
761
- nextToken = await this._retryDecodeStepWithFinitenessWindow(
762
- generatedIds,
763
- opts,
764
- `decode-step-${tokenIds.length}`
765
- );
766
- } else {
767
- throw error;
768
- }
769
- }
770
- generatedIds.push(nextToken);
771
- tokenIds.push(nextToken);
772
- this._consumeFinitenessFallbackToken(opts);
773
- if (isStopToken(nextToken, stopTokenIds, eosToken)) {
774
- break;
775
- }
776
- if (opts.stopSequences.length > 0) {
777
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
778
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1343
+ const decodeRuntime = {
1344
+ stopTokenIds,
1345
+ eosToken,
1346
+ stopSequenceStart,
1347
+ decodeToken: () => '',
1348
+ emitMode: 'token',
1349
+ };
1350
+
1351
+ if (!this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1352
+ for await (const tokenId of this._runDecodeLoop(generatedIds, opts, options, decodeRuntime)) {
1353
+ tokenIds.push(tokenId);
779
1354
  }
1355
+ } else {
1356
+ this.#state.stats.decodeTimeMs = 0;
1357
+ this.#state.stats.tokensGenerated = 1;
1358
+ this.#state.stats.decodeTokens = 1;
780
1359
  }
781
1360
 
782
- this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
783
- this.#state.stats.tokensGenerated = tokenIds.length;
784
- this.#state.stats.decodeTokens = tokenIds.length;
785
1361
  this.#state.stats.totalTimeMs = performance.now() - startTime;
786
1362
 
787
1363
  return {
@@ -804,63 +1380,72 @@ export class PipelineGenerator {
804
1380
  if (this.#state.isGenerating && options.__internalGenerate !== true) {
805
1381
  throw new Error('Generation already in progress');
806
1382
  }
1383
+ assertIncrementalDecodeSupport(this.#state, 'prefillKVOnly');
807
1384
  this._resetDecodeRuntimeState();
808
1385
  this.#state.stats.gpuTimePrefillMs = undefined;
1386
+ this.#state.stats.prefillProfileSteps = [];
809
1387
  const opts = resolvePrefillOptions(this.#state, options);
810
- const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, 'prefillKVOnly');
1388
+ const prefillStartSeqLen = this.#state.currentSeqLen;
1389
+ const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillKVOnly', opts.inputIds);
811
1390
  if (opts.debug) {
812
1391
  log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
813
1392
  }
814
1393
 
815
- let prefillResult;
816
1394
  try {
817
- prefillResult = await this._prefillToHidden(inputIds, opts);
818
- } catch (error) {
819
- if (shouldRetryWithFinitenessFallback(error)) {
820
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
821
- prefillResult = await this._retryWithFinitenessFallback(
822
- opts,
823
- 'prefillKVOnly',
824
- () => this._prefillToHidden(inputIds, opts)
825
- );
826
- } else {
827
- throw error;
1395
+ let prefillResult;
1396
+ try {
1397
+ prefillResult = await this._prefillToHidden(inputIds, opts);
1398
+ } catch (error) {
1399
+ if (this._shouldUseFinitenessFallback(error, 'prefillKVOnly')) {
1400
+ log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
1401
+ prefillResult = await this._retryWithPersistentFinitenessFallback(
1402
+ opts,
1403
+ 'prefillKVOnly',
1404
+ 1,
1405
+ () => this._prefillToHidden(inputIds, opts),
1406
+ prefillStartSeqLen
1407
+ );
1408
+ } else {
1409
+ throw error;
1410
+ }
828
1411
  }
829
- }
830
1412
 
831
- const {
832
- numTokens,
833
- startPos,
834
- currentRecorder,
835
- recordProfile,
836
- currentHiddenBuffer,
837
- } = prefillResult;
838
-
839
- // Ensure prefill work completes before returning a usable snapshot.
840
- if (currentRecorder) {
841
- await currentRecorder.submitAndWait();
842
- await recordProfile(currentRecorder);
843
- } else {
844
- const device = getDevice();
845
- if (device) {
846
- await device.queue.onSubmittedWorkDone();
1413
+ const {
1414
+ numTokens,
1415
+ startPos,
1416
+ currentRecorder,
1417
+ recordProfile,
1418
+ currentHiddenBuffer,
1419
+ } = prefillResult;
1420
+
1421
+ // Ensure prefill work completes before returning a usable snapshot.
1422
+ if (currentRecorder) {
1423
+ await currentRecorder.submitAndWait();
1424
+ await recordProfile(currentRecorder);
1425
+ } else {
1426
+ const device = getDevice();
1427
+ if (device) {
1428
+ await device.queue.onSubmittedWorkDone();
1429
+ }
847
1430
  }
848
- }
849
1431
 
850
- this.#state.currentSeqLen = startPos + numTokens;
851
- releaseBuffer(currentHiddenBuffer);
1432
+ this.#state.currentSeqLen = startPos + numTokens;
1433
+ releaseBuffer(currentHiddenBuffer);
852
1434
 
853
- const snapshot = this.#state.kvCache?.clone();
854
- if (!snapshot) {
855
- throw new Error('KV cache unavailable after prefill');
856
- }
1435
+ const snapshot = this.#state.kvCache?.clone();
1436
+ if (!snapshot) {
1437
+ throw new Error('KV cache unavailable after prefill');
1438
+ }
857
1439
 
858
- return {
859
- cache: snapshot,
860
- seqLen: this.#state.currentSeqLen,
861
- tokens: inputIds,
862
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
863
- };
1440
+ return {
1441
+ cache: snapshot,
1442
+ seqLen: this.#state.currentSeqLen,
1443
+ tokens: inputIds,
1444
+ linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
1445
+ };
1446
+ } finally {
1447
+ this._closeFinitenessFallbackWindow(opts);
1448
+ }
864
1449
  }
865
1450
 
866
1451
  async prefillWithEmbedding(prompt, options = {}) {
@@ -868,93 +1453,115 @@ export class PipelineGenerator {
868
1453
  if (this.#state.isGenerating && options.__internalGenerate !== true) {
869
1454
  throw new Error('Generation already in progress');
870
1455
  }
1456
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
871
1457
  this._resetDecodeRuntimeState();
872
1458
  this.#state.stats.gpuTimePrefillMs = undefined;
1459
+ this.#state.stats.prefillProfileSteps = [];
873
1460
  const opts = resolvePrefillEmbeddingOptions(this.#state, options);
874
- const inputIds = this._resolvePromptTokenIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding');
1461
+ const prefillStartSeqLen = this.#state.currentSeqLen;
1462
+ const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
875
1463
  if (opts.debug) {
876
1464
  log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
877
1465
  }
878
1466
 
879
- let prefillResult;
880
1467
  try {
881
- prefillResult = await this._prefillToHidden(inputIds, opts);
882
- } catch (error) {
883
- if (shouldRetryWithFinitenessFallback(error)) {
884
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
885
- prefillResult = await this._retryWithFinitenessFallback(
886
- opts,
887
- 'prefillWithEmbedding',
888
- () => this._prefillToHidden(inputIds, opts)
889
- );
1468
+ let prefillResult;
1469
+ try {
1470
+ prefillResult = await this._prefillToHidden(inputIds, opts);
1471
+ } catch (error) {
1472
+ if (shouldRetryWithFinitenessFallback(error)) {
1473
+ log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
1474
+ prefillResult = await this._retryWithPersistentFinitenessFallback(
1475
+ opts,
1476
+ 'prefillWithEmbedding',
1477
+ 1,
1478
+ () => this._prefillToHidden(inputIds, opts),
1479
+ prefillStartSeqLen
1480
+ );
1481
+ } else {
1482
+ throw error;
1483
+ }
1484
+ }
1485
+
1486
+ const {
1487
+ numTokens,
1488
+ config,
1489
+ startPos,
1490
+ activationDtype,
1491
+ activationBytes,
1492
+ currentRecorder,
1493
+ recordProfile,
1494
+ currentHiddenBuffer,
1495
+ } = prefillResult;
1496
+
1497
+ // Ensure prefill work completes before readback.
1498
+ if (currentRecorder) {
1499
+ await currentRecorder.submitAndWait();
1500
+ await recordProfile(currentRecorder);
890
1501
  } else {
891
- throw error;
1502
+ const device = getDevice();
1503
+ if (device) {
1504
+ await device.queue.onSubmittedWorkDone();
1505
+ }
892
1506
  }
893
- }
894
1507
 
895
- const {
896
- numTokens,
897
- config,
898
- startPos,
899
- activationDtype,
900
- activationBytes,
901
- currentRecorder,
902
- recordProfile,
903
- currentHiddenBuffer,
904
- } = prefillResult;
905
-
906
- // Ensure prefill work completes before readback.
907
- if (currentRecorder) {
908
- await currentRecorder.submitAndWait();
909
- await recordProfile(currentRecorder);
910
- } else {
911
- const device = getDevice();
912
- if (device) {
913
- await device.queue.onSubmittedWorkDone();
1508
+ if (!allowReadback('pipeline.prefill.embedding')) {
1509
+ throw new Error('GPU readback disabled; cannot return embedding');
914
1510
  }
915
- }
916
1511
 
917
- if (!allowReadback('pipeline.prefill.embedding')) {
918
- throw new Error('GPU readback disabled; cannot return embedding');
919
- }
1512
+ let embedding;
1513
+ try {
1514
+ const hiddenSize = config.hiddenSize;
1515
+ const hiddenBytes = numTokens * hiddenSize * activationBytes;
1516
+ const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
1517
+ if (hiddenData.byteLength === 0) {
1518
+ throw new Error('GPU readback disabled; cannot return embedding');
1519
+ }
1520
+ const hiddenStates = decodeReadback(hiddenData, activationDtype);
1521
+ const finalNormWeights = await this._getFinalNormWeights();
1522
+ embedding = this._extractEmbeddingFromHidden(
1523
+ hiddenStates,
1524
+ numTokens,
1525
+ hiddenSize,
1526
+ opts.embeddingMode,
1527
+ finalNormWeights,
1528
+ config
1529
+ );
1530
+ } finally {
1531
+ releaseBuffer(currentHiddenBuffer);
1532
+ }
920
1533
 
921
- let embedding;
922
- try {
923
- const hiddenSize = config.hiddenSize;
924
- const hiddenBytes = numTokens * hiddenSize * activationBytes;
925
- const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
926
- if (hiddenData.byteLength === 0) {
927
- throw new Error('GPU readback disabled; cannot return embedding');
1534
+ this.#state.currentSeqLen = startPos + numTokens;
1535
+
1536
+ // Batch embedding skips expensive KV cache clone and linear attention clone
1537
+ // since the caller will reset immediately after extracting the embedding.
1538
+ if (options.__skipStateSnapshot) {
1539
+ return {
1540
+ cache: null,
1541
+ seqLen: this.#state.currentSeqLen,
1542
+ tokens: inputIds,
1543
+ embedding,
1544
+ embeddingMode: opts.embeddingMode,
1545
+ linearAttention: null,
1546
+ };
928
1547
  }
929
- const hiddenStates = decodeReadback(hiddenData, activationDtype);
930
- const finalNormWeights = await this._getFinalNormWeights();
931
- embedding = this._extractEmbeddingFromHidden(
932
- hiddenStates,
933
- numTokens,
934
- hiddenSize,
935
- opts.embeddingMode,
936
- finalNormWeights,
937
- config
938
- );
939
- } finally {
940
- releaseBuffer(currentHiddenBuffer);
941
- }
942
1548
 
943
- this.#state.currentSeqLen = startPos + numTokens;
1549
+ const snapshot = this.#state.kvCache?.clone();
1550
+ if (!snapshot) {
1551
+ throw new Error('KV cache unavailable after prefill');
1552
+ }
944
1553
 
945
- const snapshot = this.#state.kvCache?.clone();
946
- if (!snapshot) {
947
- throw new Error('KV cache unavailable after prefill');
1554
+ return {
1555
+ cache: snapshot,
1556
+ seqLen: this.#state.currentSeqLen,
1557
+ tokens: inputIds,
1558
+ embedding,
1559
+ embeddingMode: opts.embeddingMode,
1560
+ linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
1561
+ };
1562
+ } finally {
1563
+ this._closeFinitenessFallbackWindow(opts);
948
1564
  }
949
-
950
- return {
951
- cache: snapshot,
952
- seqLen: this.#state.currentSeqLen,
953
- tokens: inputIds,
954
- embedding,
955
- embeddingMode: opts.embeddingMode,
956
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
957
- };
958
1565
  }
959
1566
 
960
1567
  async prefillWithLogits(prompt, options = {}) {
@@ -962,29 +1569,36 @@ export class PipelineGenerator {
962
1569
  if (this.#state.isGenerating && options.__internalGenerate !== true) {
963
1570
  throw new Error('Generation already in progress');
964
1571
  }
1572
+ assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
965
1573
  this._resetDecodeRuntimeState();
966
1574
  this.#state.stats.gpuTimePrefillMs = undefined;
1575
+ this.#state.stats.prefillProfileSteps = [];
967
1576
  const opts = resolvePrefillOptions(this.#state, options);
968
- const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
1577
+ try {
1578
+ const { inputIds, logits } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
969
1579
 
970
- const snapshot = this.#state.kvCache?.clone();
971
- if (!snapshot) {
972
- throw new Error('KV cache unavailable after prefill');
973
- }
1580
+ const snapshot = this.#state.kvCache?.clone();
1581
+ if (!snapshot) {
1582
+ throw new Error('KV cache unavailable after prefill');
1583
+ }
974
1584
 
975
- return {
976
- cache: snapshot,
977
- seqLen: this.#state.currentSeqLen,
978
- tokens: inputIds,
979
- logits,
980
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
981
- };
1585
+ return {
1586
+ cache: snapshot,
1587
+ seqLen: this.#state.currentSeqLen,
1588
+ tokens: inputIds,
1589
+ logits,
1590
+ linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
1591
+ };
1592
+ } finally {
1593
+ this._closeFinitenessFallbackWindow(opts);
1594
+ }
982
1595
  }
983
1596
 
984
1597
 
985
1598
  async *generateWithPrefixKV(prefix, prompt, options = {}) {
986
1599
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
987
1600
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
1601
+ assertIncrementalDecodeSupport(this.#state, 'generateWithPrefixKV');
988
1602
 
989
1603
  validateCallTimeOptions(options);
990
1604
 
@@ -1016,6 +1630,7 @@ export class PipelineGenerator {
1016
1630
  resetActiveExecutionPlan(this.#state);
1017
1631
  this.#state.stats.gpuTimePrefillMs = undefined;
1018
1632
  this.#state.stats.gpuTimeDecodeMs = undefined;
1633
+ this.#state.stats.prefillProfileSteps = [];
1019
1634
  this.#state.decodeRing?.reset();
1020
1635
  this.#state.stats.decodeRecordMs = 0;
1021
1636
  this.#state.stats.decodeSubmitWaitMs = 0;
@@ -1048,9 +1663,15 @@ export class PipelineGenerator {
1048
1663
  seed: opts.seed,
1049
1664
  });
1050
1665
 
1666
+ const decodeRuntime = {
1667
+ stopTokenIds: this.#state.modelConfig.stopTokenIds,
1668
+ eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
1669
+ stopSequenceStart: promptTokenCount,
1670
+ decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
1671
+ logBatchPath: false,
1672
+ };
1051
1673
  generatedIds.push(firstToken);
1052
1674
  this.#state.stats.ttftMs = performance.now() - startTime;
1053
-
1054
1675
  const firstText = resolveTokenText(
1055
1676
  this.#state.tokenizer,
1056
1677
  [firstToken],
@@ -1061,13 +1682,13 @@ export class PipelineGenerator {
1061
1682
  yield firstText;
1062
1683
  if (options.onToken) options.onToken(firstToken, firstText);
1063
1684
 
1064
- yield* this._runDecodeLoop(generatedIds, opts, options, {
1065
- stopTokenIds: this.#state.modelConfig.stopTokenIds,
1066
- eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
1067
- stopSequenceStart: promptTokenCount,
1068
- decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
1069
- logBatchPath: false,
1070
- });
1685
+ if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1686
+ this.#state.stats.decodeTimeMs = 0;
1687
+ this.#state.stats.tokensGenerated = 1;
1688
+ this.#state.stats.decodeTokens = 1;
1689
+ } else {
1690
+ yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
1691
+ }
1071
1692
  this.#state.stats.totalTimeMs = performance.now() - startTime;
1072
1693
  } finally {
1073
1694
  this._closeFinitenessFallbackWindow(opts);
@@ -1093,6 +1714,20 @@ export class PipelineGenerator {
1093
1714
  let tokensGenerated = 1;
1094
1715
  markKernelCacheWarmed();
1095
1716
 
1717
+ // Step 4: Lazily initialise PLE buffer cache for decode-path slice reuse.
1718
+ const pleHiddenSize = Number(this.#state.modelConfig.hiddenSizePerLayerInput ?? 0);
1719
+ if (pleHiddenSize > 0 && !this.#state.pleCache) {
1720
+ const activationDtype = resolveActiveExecutionPlan(this.#state).activationDtype;
1721
+ const bpe = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
1722
+ this.#state.pleCache = createPleBufferCache(
1723
+ this.#state.modelConfig.numLayers,
1724
+ pleHiddenSize * bpe,
1725
+ );
1726
+ }
1727
+ if (pleHiddenSize > 0) {
1728
+ await primePleDecodeRuntimeCache(this.#state, generatedIds);
1729
+ }
1730
+
1096
1731
  const decodeStart = performance.now();
1097
1732
  const lmHead = this.#state.weights.get('lm_head');
1098
1733
  const embedBuffer = this.#state.weights.get('embed');
@@ -1101,20 +1736,42 @@ export class PipelineGenerator {
1101
1736
  || lmHead instanceof Float32Array
1102
1737
  || embedBuffer instanceof Float32Array;
1103
1738
  const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
1739
+ const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
1104
1740
  const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
1105
- const executionPlan = opts.executionPlan;
1106
- let useBatchPath = shouldUseBatchDecode({
1107
- batchSize: executionPlan.batchSize,
1108
- useGPU: this.#state.useGPU,
1109
- gpuSamplingAvailable,
1110
- disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
1111
- disableCommandBatching: executionPlan.disableCommandBatching,
1112
- isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
1113
- finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
1741
+ const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
1742
+ config: this.#state.modelConfig,
1743
+ weights: this.#state.weights,
1744
+ });
1745
+ const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
1746
+ config: this.#state.modelConfig,
1747
+ weights: this.#state.weights,
1748
+ });
1749
+ const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
1750
+ const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
1751
+ hasRangeBackedPerLayerInputs,
1752
+ pleHotVocabularyRuntime,
1753
+ tokenId: generatedIds[generatedIds.length - 1] ?? null,
1114
1754
  });
1755
+ const initialHotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
1756
+ const executionPlan = opts.executionPlan;
1757
+ let useBatchPath = replayPrefillDecode
1758
+ ? false
1759
+ : shouldUseBatchDecode({
1760
+ batchSize: executionPlan.batchSize,
1761
+ useGPU: this.#state.useGPU,
1762
+ gpuSamplingAvailable,
1763
+ disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
1764
+ disableCommandBatching: executionPlan.disableCommandBatching,
1765
+ isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
1766
+ finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
1767
+ hasLinearAttentionLayers: hasLinearLayers,
1768
+ selfSpeculationEnabled: opts.speculation?.mode === 'self' && !initialHotVocabularyBatchDecodeAvailable,
1769
+ hasRangeBackedPerLayerInputs,
1770
+ });
1115
1771
  if (!useBatchPath) {
1116
1772
  let reason = null;
1117
- if (hasCpuWeights) reason = 'cpu_weights';
1773
+ if (replayPrefillDecode) reason = 'replay_prefill_decode';
1774
+ else if (hasCpuWeights) reason = 'cpu_weights';
1118
1775
  else if (!this.#state.useGPU) reason = 'no_gpu';
1119
1776
  else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
1120
1777
  else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
@@ -1122,10 +1779,15 @@ export class PipelineGenerator {
1122
1779
  else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
1123
1780
  else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
1124
1781
  else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
1125
- this.#state.stats.decodeMode = opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token';
1782
+ else if (hasLinearLayers) reason = 'linear_attention_layers';
1783
+ this.#state.stats.decodeMode = replayPrefillDecode
1784
+ ? 'replay_prefill'
1785
+ : (opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token');
1126
1786
  this.#state.stats.batchGuardReason = reason;
1127
1787
  } else {
1128
- this.#state.stats.decodeMode = 'batched_gpu';
1788
+ this.#state.stats.decodeMode = hasRangeBackedPerLayerInputs
1789
+ ? 'batched_gpu_stepwise_ple'
1790
+ : 'batched_gpu';
1129
1791
  this.#state.stats.batchGuardReason = null;
1130
1792
  }
1131
1793
 
@@ -1147,10 +1809,20 @@ export class PipelineGenerator {
1147
1809
  if (this._hasFinitenessFallbackWindow() && useBatchPath) {
1148
1810
  useBatchPath = false;
1149
1811
  }
1812
+ const hotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
1150
1813
 
1151
1814
  if (useBatchPath) {
1152
1815
  const remaining = opts.maxTokens - tokensGenerated;
1153
- const thisBatchSize = Math.min(executionPlan.batchSize * intervalBatches, remaining);
1816
+ const maxBatchDecodeTokens = resolveMaxBatchDecodeTokens({
1817
+ hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
1818
+ hasGpuSplitPerLayerInputs,
1819
+ hasLinearAttentionLayers: hasLinearLayers,
1820
+ });
1821
+ const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
1822
+ const boundedBatchTokens = maxBatchDecodeTokens == null
1823
+ ? requestedBatchTokens
1824
+ : Math.min(requestedBatchTokens, maxBatchDecodeTokens);
1825
+ const thisBatchSize = Math.min(boundedBatchTokens, remaining);
1154
1826
  const lastToken = generatedIds[generatedIds.length - 1];
1155
1827
 
1156
1828
  try {
@@ -1180,11 +1852,19 @@ export class PipelineGenerator {
1180
1852
  }
1181
1853
  }
1182
1854
  if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
1183
- if (hitStop || batchResult.actualCount < thisBatchSize) break;
1184
1855
  if (opts.stopSequences.length > 0) {
1185
1856
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1186
1857
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1187
1858
  }
1859
+ if (hitStop) break;
1860
+ if (shouldDisableBatchDecodeAfterShortBatch({
1861
+ hitStop,
1862
+ actualCount: batchResult.actualCount,
1863
+ requestedCount: thisBatchSize,
1864
+ })) {
1865
+ useBatchPath = false;
1866
+ continue;
1867
+ }
1188
1868
  } catch (error) {
1189
1869
  log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
1190
1870
  useBatchPath = false;
@@ -1192,7 +1872,7 @@ export class PipelineGenerator {
1192
1872
  try {
1193
1873
  nextToken = await decodeSingleTokenViaLogits();
1194
1874
  } catch (singleTokenError) {
1195
- if (shouldRetryWithFinitenessFallback(singleTokenError)) {
1875
+ if (this._shouldUseFinitenessFallback(singleTokenError, `decode-batch-step-${tokensGenerated}`)) {
1196
1876
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
1197
1877
  nextToken = await this._retryDecodeStepWithFinitenessWindow(
1198
1878
  generatedIds,
@@ -1217,11 +1897,13 @@ export class PipelineGenerator {
1217
1897
  if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
1218
1898
  }
1219
1899
  } else if (opts.speculation?.mode === 'self') {
1220
- // Self-speculation: decode 2 tokens per iteration (base + speculative).
1221
- // Same-model speculation always accepts under greedy because the model
1222
- // is deterministic — both base and speculative use the same weights and
1223
- // state. The benefit is amortizing per-iteration overhead for models
1224
- // where batch decode is disabled (e.g., linear attention).
1900
+ // Self-speculation: decode one base token plus a configurable burst of
1901
+ // speculative tokens per iteration. Same-model speculation always
1902
+ // accepts under greedy because the model is deterministic — both base
1903
+ // and speculative use the same weights and state. The benefit is
1904
+ // amortizing per-iteration overhead for models where batch decode is
1905
+ // disabled (e.g., linear attention).
1906
+ const speculativeBurstTokens = Math.max(1, Math.trunc(opts.speculation?.tokens ?? 1));
1225
1907
  const doSpecDecode = hasLinearLayers
1226
1908
  ? () => this._decodeStep(generatedIds, opts)
1227
1909
  : decodeSingleTokenViaLogits;
@@ -1231,7 +1913,7 @@ export class PipelineGenerator {
1231
1913
  try {
1232
1914
  baseToken = await doSpecDecode();
1233
1915
  } catch (error) {
1234
- if (shouldRetryWithFinitenessFallback(error)) {
1916
+ if (this._shouldUseFinitenessFallback(error, `spec-base-${tokensGenerated}`)) {
1235
1917
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
1236
1918
  baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
1237
1919
  } else {
@@ -1257,39 +1939,52 @@ export class PipelineGenerator {
1257
1939
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1258
1940
  }
1259
1941
 
1260
- // Speculative decode (tokens=1)
1261
- let specToken;
1262
- try {
1263
- specToken = await doSpecDecode();
1264
- } catch (error) {
1265
- if (shouldRetryWithFinitenessFallback(error)) {
1266
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
1267
- specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
1942
+ for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
1943
+ if (tokensGenerated >= opts.maxTokens) {
1944
+ break;
1945
+ }
1946
+ let specToken;
1947
+ try {
1948
+ specToken = await doSpecDecode();
1949
+ } catch (error) {
1950
+ if (this._shouldUseFinitenessFallback(error, `spec-extra-${tokensGenerated}`)) {
1951
+ log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
1952
+ specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
1953
+ } else {
1954
+ throw error;
1955
+ }
1956
+ }
1957
+ generatedIds.push(specToken);
1958
+ tokensGenerated++;
1959
+ this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
1960
+ this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
1961
+ if (emitMode === 'token') {
1962
+ yield specToken;
1963
+ if (options.onToken) options.onToken(specToken, '');
1268
1964
  } else {
1269
- throw error;
1965
+ const text = decodeToken(specToken);
1966
+ yield text;
1967
+ if (options.onToken) options.onToken(specToken, text);
1270
1968
  }
1271
- }
1272
- generatedIds.push(specToken);
1273
- tokensGenerated++;
1274
- this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
1275
- this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
1276
- if (emitMode === 'token') {
1277
- yield specToken;
1278
- if (options.onToken) options.onToken(specToken, '');
1279
- } else {
1280
- const text = decodeToken(specToken);
1281
- yield text;
1282
- if (options.onToken) options.onToken(specToken, text);
1283
- }
1284
- this._consumeFinitenessFallbackToken(opts);
1969
+ this._consumeFinitenessFallbackToken(opts);
1285
1970
 
1286
- if (opts.debug || opts.benchmark) {
1287
- const elapsedMs = performance.now() - decodeStart;
1288
- const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
1289
- log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
1290
- }
1971
+ if (opts.debug || opts.benchmark) {
1972
+ const elapsedMs = performance.now() - decodeStart;
1973
+ const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
1974
+ log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
1975
+ }
1291
1976
 
1292
- if (isStopToken(specToken, stopTokenIds, eosToken)) break;
1977
+ if (isStopToken(specToken, stopTokenIds, eosToken)) {
1978
+ break;
1979
+ }
1980
+ if (opts.stopSequences.length > 0) {
1981
+ const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1982
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1983
+ break;
1984
+ }
1985
+ }
1986
+ }
1987
+ if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) break;
1293
1988
  if (opts.stopSequences.length > 0) {
1294
1989
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1295
1990
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
@@ -1302,7 +1997,7 @@ export class PipelineGenerator {
1302
1997
  ? await this._decodeStep(generatedIds, opts)
1303
1998
  : await decodeSingleTokenViaLogits();
1304
1999
  } catch (error) {
1305
- if (shouldRetryWithFinitenessFallback(error)) {
2000
+ if (this._shouldUseFinitenessFallback(error, `decode-step-${tokensGenerated}`)) {
1306
2001
  log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
1307
2002
  nextToken = await this._retryDecodeStepWithFinitenessWindow(
1308
2003
  generatedIds,
@@ -1316,6 +2011,20 @@ export class PipelineGenerator {
1316
2011
  const tokenTime = performance.now() - tokenStart;
1317
2012
  generatedIds.push(nextToken);
1318
2013
  tokensGenerated++;
2014
+
2015
+ // Step 5: Fire-and-forget prefetch of next token's PLE row.
2016
+ if (pleHiddenSize > 0) {
2017
+ const pleWeights = this.#state.weights.get('per_layer_inputs');
2018
+ if (pleWeights?.embedTokensPerLayer) {
2019
+ this.#state.plePrefetchPending = prefetchPerLayerRow(
2020
+ nextToken,
2021
+ pleWeights.embedTokensPerLayer,
2022
+ this.#state.modelConfig.numLayers * pleHiddenSize,
2023
+ this.#state.runtimeConfig.inference.session?.perLayerInputs ?? this.#state.modelConfig.perLayerInputsSession,
2024
+ );
2025
+ }
2026
+ }
2027
+
1319
2028
  const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
1320
2029
  if (emitMode === 'token') {
1321
2030
  yield nextToken;
@@ -1356,7 +2065,27 @@ export class PipelineGenerator {
1356
2065
  const config = this.#state.modelConfig;
1357
2066
  const startPos = this.#state.currentSeqLen;
1358
2067
  const returnHidden = opts?._returnHidden === true;
2068
+ const embeddingInputIds = resolvePrefillEmbeddingInputIds(
2069
+ inputIds,
2070
+ opts?.embeddingInputSpan ?? null,
2071
+ '_prefill'
2072
+ );
2073
+ const multimodalBidirectionalSpan = resolvePrefillMultimodalBidirectionalSpan(
2074
+ inputIds,
2075
+ opts?.multimodalBidirectionalSpan ?? null,
2076
+ '_prefill'
2077
+ );
2078
+ if (embeddingInputIds !== inputIds) {
2079
+ this._assertTokenIdsInRange(embeddingInputIds, '_prefill.embeddingInputIds');
2080
+ }
2081
+ const embeddingOverride = normalizePrefixEmbeddingOverride(
2082
+ opts?.embeddingOverrides ?? null,
2083
+ config.hiddenSize,
2084
+ numTokens,
2085
+ '_prefill'
2086
+ );
1359
2087
  this.#state.stats.gpuTimePrefillMs = undefined;
2088
+ this.#state.stats.prefillProfileSteps = [];
1360
2089
 
1361
2090
  if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
1362
2091
  this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
@@ -1384,9 +2113,11 @@ export class PipelineGenerator {
1384
2113
 
1385
2114
  const device = getDevice();
1386
2115
  const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
1387
- const disableCommandBatching = opts.disableCommandBatching === true
1388
- || opts.debug === true
1389
- || this.#state.kvCache?.layout === 'bdpa_paged';
2116
+ const disableCommandBatching = shouldDisablePrefillCommandBatching(
2117
+ this.#state,
2118
+ opts,
2119
+ multimodalBidirectionalSpan
2120
+ );
1390
2121
  const createRecorder = (label) => {
1391
2122
  if (!device || disableCommandBatching) return undefined;
1392
2123
  return opts.profile ? createProfilingRecorder(label) : createCommandRecorder(label);
@@ -1405,6 +2136,12 @@ export class PipelineGenerator {
1405
2136
  opts.executionPlan
1406
2137
  );
1407
2138
  context.currentTokenIds = inputIds;
2139
+ context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
2140
+ ? null
2141
+ : {
2142
+ start: startPos + multimodalBidirectionalSpan.offset,
2143
+ length: multimodalBidirectionalSpan.length,
2144
+ };
1408
2145
  let gpuTimePrefillMs = 0;
1409
2146
  let hasGpuTimePrefill = false;
1410
2147
  const recordProfile = async (rec) => {
@@ -1416,6 +2153,11 @@ export class PipelineGenerator {
1416
2153
  hasGpuTimePrefill = true;
1417
2154
  }
1418
2155
  if (timings) {
2156
+ recordPrefillProfileStep(this.#state, {
2157
+ label: rec.label,
2158
+ timings,
2159
+ totalMs: total ?? undefined,
2160
+ });
1419
2161
  log.warn('Profile', `Prefill (${rec.label}):`);
1420
2162
  log.warn('Profile', CommandRecorder.formatProfileReport(timings));
1421
2163
  }
@@ -1427,9 +2169,23 @@ export class PipelineGenerator {
1427
2169
  resetSubmitStats();
1428
2170
  }
1429
2171
 
2172
+ const preserveBufferAcrossRecorderSubmit = (buffer, activeRecorder, label) => {
2173
+ if (!activeRecorder || !isGpuBufferInstance(buffer)) {
2174
+ return buffer;
2175
+ }
2176
+ const carryBuffer = acquireBuffer(
2177
+ buffer.size,
2178
+ typeof buffer.usage === 'number' ? buffer.usage : undefined,
2179
+ label
2180
+ );
2181
+ activeRecorder.getEncoder().copyBufferToBuffer(buffer, 0, carryBuffer, 0, buffer.size);
2182
+ activeRecorder.trackTemporaryBuffer(buffer);
2183
+ return carryBuffer;
2184
+ };
2185
+
1430
2186
  const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
1431
2187
  const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
1432
- let hiddenStates = await embed(inputIds, embedBuffer, {
2188
+ let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
1433
2189
  hiddenSize: config.hiddenSize,
1434
2190
  vocabSize: config.vocabSize,
1435
2191
  scaleEmbeddings: config.scaleEmbeddings,
@@ -1437,12 +2193,53 @@ export class PipelineGenerator {
1437
2193
  recorder,
1438
2194
  transpose: this.#state.embeddingTranspose,
1439
2195
  debugProbes: this.#state.runtimeConfig.shared.debug.probes,
2196
+ operatorDiagnostics: this.#state.operatorDiagnostics,
1440
2197
  activationDtype,
1441
2198
  embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
2199
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1442
2200
  });
2201
+ let hiddenStates = baseEmbeddings;
2202
+ let perLayerInputs = null;
2203
+ try {
2204
+ hiddenStates = await applyPrefixEmbeddingOverride(
2205
+ baseEmbeddings,
2206
+ embeddingOverride,
2207
+ config.hiddenSize,
2208
+ '_prefill',
2209
+ this.#state.executionV1State?.policies ?? null
2210
+ );
2211
+ perLayerInputs = await preparePerLayerInputs(
2212
+ embeddingInputIds,
2213
+ embeddingInputIds === inputIds ? hiddenStates : baseEmbeddings,
2214
+ context,
2215
+ {
2216
+ numTokens,
2217
+ }
2218
+ );
2219
+ } catch (error) {
2220
+ if (isGpuBufferInstance(hiddenStates?.buffer)) {
2221
+ releaseBuffer(hiddenStates.buffer);
2222
+ }
2223
+ if (hiddenStates === baseEmbeddings) {
2224
+ baseEmbeddings = null;
2225
+ }
2226
+ hiddenStates = null;
2227
+ throw error;
2228
+ } finally {
2229
+ if (hiddenStates !== baseEmbeddings && isGpuBufferInstance(baseEmbeddings?.buffer)) {
2230
+ releaseBuffer(baseEmbeddings.buffer);
2231
+ }
2232
+ baseEmbeddings = null;
2233
+ }
1443
2234
 
1444
2235
  if (opts.debug && isGpuBufferInstance(hiddenStates)) {
1445
2236
  if (recorder) {
2237
+ hiddenStates = createTensor(
2238
+ preserveBufferAcrossRecorderSubmit(hiddenStates.buffer, recorder, 'prefill_embed_carry'),
2239
+ hiddenStates.dtype,
2240
+ hiddenStates.shape,
2241
+ hiddenStates.label
2242
+ );
1446
2243
  await recorder.submitAndWait();
1447
2244
  await recordProfile(recorder);
1448
2245
  }
@@ -1473,67 +2270,123 @@ export class PipelineGenerator {
1473
2270
 
1474
2271
  let currentRecorder = recorder;
1475
2272
 
1476
- let currentHiddenBuffer = hiddenStates.buffer;
1477
- for (let l = 0; l < config.numLayers; l++) {
1478
- context.recorder = currentRecorder;
2273
+ // Chunked recorder submission: submit every N layers to release tracked intermediate
2274
+ // buffers, preventing unbounded memory growth during large prefills. Critical for
2275
+ // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2276
+ const PREFILL_RECORDER_CHUNK_LAYERS = 4;
1479
2277
 
1480
- const prevBuffer = currentHiddenBuffer;
1481
- const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
1482
- if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
1483
- currentHiddenBuffer = layerOutput;
1484
-
1485
- const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
1486
-
1487
- if (isCheckpoint && currentRecorder) {
1488
- await currentRecorder.submitAndWait();
1489
- await recordProfile(currentRecorder);
1490
- currentRecorder = undefined;
1491
- }
2278
+ let currentHiddenBuffer = hiddenStates.buffer;
2279
+ try {
2280
+ for (let l = 0; l < config.numLayers; l++) {
2281
+ context.recorder = currentRecorder;
2282
+ context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
2283
+
2284
+ const prevBuffer = currentHiddenBuffer;
2285
+ const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
2286
+ if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
2287
+ currentHiddenBuffer = layerOutput;
2288
+ releasePerLayerInputBuffer(
2289
+ context.perLayerInputBuffer,
2290
+ currentRecorder,
2291
+ context.decodeBuffers,
2292
+ this.#state.pleCache ?? null
2293
+ );
2294
+ if (perLayerInputs) {
2295
+ perLayerInputs[l] = null;
2296
+ }
2297
+ context.perLayerInputBuffer = null;
2298
+
2299
+ const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
2300
+ const isChunkBoundary = !isCheckpoint
2301
+ && currentRecorder
2302
+ && l < config.numLayers - 1
2303
+ && (l + 1) % PREFILL_RECORDER_CHUNK_LAYERS === 0;
2304
+
2305
+ if (isCheckpoint && currentRecorder) {
2306
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2307
+ currentHiddenBuffer,
2308
+ currentRecorder,
2309
+ 'prefill_checkpoint_carry'
2310
+ );
2311
+ await currentRecorder.submitAndWait();
2312
+ await recordProfile(currentRecorder);
2313
+ currentRecorder = undefined;
2314
+ }
1492
2315
 
1493
- const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
1494
- if (shouldDebug && !currentRecorder) {
1495
- const device = getDevice();
1496
- if (device) {
1497
- if (allowReadback(`pipeline.prefill.layer-${l}`)) {
1498
- try {
1499
- const sampleSize = config.hiddenSize * activationBytes;
1500
- const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
1501
- const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
1502
- const data = decodeReadback(readback, activationDtype);
1503
- let min = Infinity;
1504
- let max = -Infinity;
1505
- let maxAbs = 0;
1506
- for (const v of data) {
1507
- if (!Number.isFinite(v)) continue;
1508
- if (v < min) min = v;
1509
- if (v > max) max = v;
1510
- const av = Math.abs(v);
1511
- if (av > maxAbs) maxAbs = av;
2316
+ const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
2317
+ if (shouldDebug && !currentRecorder) {
2318
+ const device = getDevice();
2319
+ if (device) {
2320
+ if (allowReadback(`pipeline.prefill.layer-${l}`)) {
2321
+ try {
2322
+ const sampleSize = config.hiddenSize * activationBytes;
2323
+ const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
2324
+ const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
2325
+ const data = decodeReadback(readback, activationDtype);
2326
+ let min = Infinity;
2327
+ let max = -Infinity;
2328
+ let maxAbs = 0;
2329
+ for (const v of data) {
2330
+ if (!Number.isFinite(v)) continue;
2331
+ if (v < min) min = v;
2332
+ if (v > max) max = v;
2333
+ const av = Math.abs(v);
2334
+ if (av > maxAbs) maxAbs = av;
2335
+ }
2336
+ const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
2337
+ log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
2338
+ } catch (e) {
2339
+ log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
1512
2340
  }
1513
- const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
1514
- log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
1515
- } catch (e) {
1516
- log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
1517
2341
  }
1518
2342
  }
1519
2343
  }
1520
- }
1521
2344
 
1522
- if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
1523
- currentRecorder = createRecorder('prefill-cont');
1524
- }
2345
+ if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
2346
+ currentRecorder = createRecorder('prefill-cont');
2347
+ }
1525
2348
 
1526
- if (prevBuffer !== currentHiddenBuffer) {
1527
- if (currentRecorder) {
1528
- currentRecorder.trackTemporaryBuffer(prevBuffer);
1529
- } else {
1530
- releaseBuffer(prevBuffer);
2349
+ if (prevBuffer !== currentHiddenBuffer) {
2350
+ if (currentRecorder) {
2351
+ currentRecorder.trackTemporaryBuffer(prevBuffer);
2352
+ } else {
2353
+ releaseBuffer(prevBuffer);
2354
+ }
2355
+ }
2356
+
2357
+ if (isChunkBoundary) {
2358
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2359
+ currentHiddenBuffer,
2360
+ currentRecorder,
2361
+ 'prefill_chunk_carry'
2362
+ );
2363
+ await currentRecorder.submitAndWait();
2364
+ await recordProfile(currentRecorder);
2365
+ currentRecorder = createRecorder('prefill-chunk');
2366
+ }
2367
+ }
2368
+ } finally {
2369
+ context.perLayerInputBuffer = null;
2370
+ if (perLayerInputs) {
2371
+ for (const buffer of perLayerInputs) {
2372
+ releasePerLayerInputBuffer(
2373
+ buffer,
2374
+ currentRecorder,
2375
+ context.decodeBuffers,
2376
+ this.#state.pleCache ?? null
2377
+ );
1531
2378
  }
1532
2379
  }
2380
+ releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
1533
2381
  }
1534
2382
 
1535
2383
  if (this.#state.finitenessBuffer) {
1536
2384
  if (currentRecorder) {
2385
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2386
+ currentHiddenBuffer,
2387
+ currentRecorder,
2388
+ 'prefill_finiteness_carry'
2389
+ );
1537
2390
  await currentRecorder.submitAndWait();
1538
2391
  await recordProfile(currentRecorder);
1539
2392
  currentRecorder = undefined;
@@ -1574,6 +2427,13 @@ export class PipelineGenerator {
1574
2427
  }
1575
2428
 
1576
2429
  if (returnHidden) {
2430
+ if (currentRecorder) {
2431
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2432
+ currentHiddenBuffer,
2433
+ currentRecorder,
2434
+ 'prefill_return_hidden_carry'
2435
+ );
2436
+ }
1577
2437
  return {
1578
2438
  numTokens,
1579
2439
  config,
@@ -1603,7 +2463,8 @@ export class PipelineGenerator {
1603
2463
  currentHiddenBuffer,
1604
2464
  numTokens,
1605
2465
  getLogitsWeights(this.#state),
1606
- getLogitsConfig(this.#state)
2466
+ getLogitsConfig(this.#state),
2467
+ this.#state.operatorDiagnostics
1607
2468
  );
1608
2469
  logitsVocabSize = recorded.vocabSize;
1609
2470
  usedRecordedLogits = true;
@@ -1636,7 +2497,8 @@ export class PipelineGenerator {
1636
2497
  undefined,
1637
2498
  debugCheckBuffer,
1638
2499
  this.#state.runtimeConfig.shared.debug.probes,
1639
- { lastPositionOnly: true }
2500
+ { lastPositionOnly: true },
2501
+ this.#state.operatorDiagnostics
1640
2502
  );
1641
2503
  const fallbackHealth = getLogitsHealth(fallbackLogits);
1642
2504
  if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
@@ -1655,6 +2517,11 @@ export class PipelineGenerator {
1655
2517
  releaseBuffer(currentHiddenBuffer);
1656
2518
  } else {
1657
2519
  if (currentRecorder) {
2520
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2521
+ currentHiddenBuffer,
2522
+ currentRecorder,
2523
+ 'prefill_logits_carry'
2524
+ );
1658
2525
  await currentRecorder.submitAndWait();
1659
2526
  await recordProfile(currentRecorder);
1660
2527
  }
@@ -1668,7 +2535,8 @@ export class PipelineGenerator {
1668
2535
  undefined,
1669
2536
  debugCheckBuffer,
1670
2537
  this.#state.runtimeConfig.shared.debug.probes,
1671
- { lastPositionOnly: true }
2538
+ { lastPositionOnly: true },
2539
+ this.#state.operatorDiagnostics
1672
2540
  );
1673
2541
 
1674
2542
  lastLogits = logits.length === logitsVocabSize
@@ -1708,6 +2576,10 @@ export class PipelineGenerator {
1708
2576
 
1709
2577
 
1710
2578
  async _decodeStep(currentIds, opts) {
2579
+ if (usesReplayPrefillDecode(this.#state)) {
2580
+ const stepResult = await this._decodeStepToLogits(currentIds, opts);
2581
+ return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
2582
+ }
1711
2583
  const debugCheckBuffer = this.#state.debug
1712
2584
  ? (buffer, label, numTokens, expectedDim) =>
1713
2585
  debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
@@ -1732,6 +2604,7 @@ export class PipelineGenerator {
1732
2604
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
1733
2605
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
1734
2606
  resetActiveExecutionPlan(this.#state);
2607
+ assertIncrementalDecodeSupport(this.#state, 'advanceWithToken');
1735
2608
 
1736
2609
  validateCallTimeOptions(options);
1737
2610
 
@@ -1749,6 +2622,7 @@ export class PipelineGenerator {
1749
2622
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
1750
2623
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
1751
2624
  resetActiveExecutionPlan(this.#state);
2625
+ assertIncrementalDecodeSupport(this.#state, 'advanceWithTokenAndEmbedding');
1752
2626
 
1753
2627
  validateCallTimeOptions(options);
1754
2628