doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -3,6 +3,7 @@ import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
3
3
  import { recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
4
4
  import { recordRepPenalty } from '../../../gpu/kernels/rep-penalty.js';
5
5
  import { recordCheckStop } from '../../../gpu/kernels/check-stop.js';
6
+ import { recordCheckHotVocabStop } from '../../../gpu/kernels/check-hot-vocab-stop.js';
6
7
  import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
7
8
  import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
8
9
  import { allowReadback } from '../../../gpu/perf-guards.js';
@@ -26,6 +27,13 @@ import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-run
26
27
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
27
28
  import { hasLinearAttentionLayers } from './linear-attention.js';
28
29
  import { hasConvLayers } from './layer.js';
30
+ import {
31
+ preparePerLayerInputs,
32
+ createPleBufferCache,
33
+ prefetchPerLayerRow,
34
+ hasRangeBackedPerLayerInputEmbeddings,
35
+ getPleHotVocabularyRuntime,
36
+ } from './per-layer-inputs.js';
29
37
 
30
38
  const UNKNOWN_TOKEN_TEXT = '<unknown>';
31
39
 
@@ -71,6 +79,64 @@ function isOwnedDecodeBuffer(candidate, decodeHiddenBuffer, decodeAltBuffer) {
71
79
  return candidate === decodeAltBuffer;
72
80
  }
73
81
 
82
+ function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
83
+ if (!buffer) {
84
+ return;
85
+ }
86
+ const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
87
+ if (ownsBuffer) {
88
+ return;
89
+ }
90
+ const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
91
+ if (cachedPleBuffer) {
92
+ return;
93
+ }
94
+ if (recorder) {
95
+ recorder.trackTemporaryBuffer(buffer);
96
+ return;
97
+ }
98
+ releaseBuffer(buffer);
99
+ }
100
+
101
+ function schedulePlePrefetchForToken(state, tokenId) {
102
+ if (state?.prefetchPleNextToken !== true) {
103
+ return;
104
+ }
105
+ const config = state.modelConfig;
106
+ const pleHiddenSize = Number(config?.hiddenSizePerLayerInput ?? 0);
107
+ if (!Number.isFinite(pleHiddenSize) || pleHiddenSize <= 0) {
108
+ return;
109
+ }
110
+ const pleWeights = state.weights.get('per_layer_inputs');
111
+ if (!pleWeights?.embedTokensPerLayer) {
112
+ return;
113
+ }
114
+ const perLayerInputsSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
115
+ state.plePrefetchPending = prefetchPerLayerRow(
116
+ tokenId,
117
+ pleWeights.embedTokensPerLayer,
118
+ config.numLayers * pleHiddenSize,
119
+ perLayerInputsSession
120
+ );
121
+ }
122
+
123
+ function getReusableSampleReadbackBuffer(state, device, size) {
124
+ const existing = state.sampleReadbackBuffer;
125
+ if (existing && existing.size >= size) {
126
+ return existing;
127
+ }
128
+ if (existing) {
129
+ existing.destroy();
130
+ }
131
+ const buffer = device.createBuffer({
132
+ label: 'sample_staging_reuse',
133
+ size,
134
+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
135
+ });
136
+ state.sampleReadbackBuffer = buffer;
137
+ return buffer;
138
+ }
139
+
74
140
  export class FinitenessError extends Error {
75
141
  constructor(message) {
76
142
  super(message);
@@ -144,11 +210,15 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
144
210
  const ownsStagingBuffer = options.ownsStagingBuffer === true;
145
211
  const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
146
212
  const ring = options.ring ?? null;
213
+ const cleanupRecorder = options.cleanupRecorder ?? null;
147
214
  let mapped = false;
215
+ let cleanupCompleted = false;
148
216
 
149
217
  try {
150
218
  await stagingBuffer.mapAsync(GPUMapMode.READ);
151
219
  mapped = true;
220
+ cleanupRecorder?.completeDeferredCleanup();
221
+ cleanupCompleted = true;
152
222
  const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
153
223
  return {
154
224
  nextToken: mappedWords[0],
@@ -160,6 +230,9 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
160
230
  if (mapped) {
161
231
  stagingBuffer.unmap();
162
232
  }
233
+ if (!cleanupCompleted) {
234
+ cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
235
+ }
163
236
  if (ownsStagingBuffer) {
164
237
  stagingBuffer.destroy();
165
238
  }
@@ -167,6 +240,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
167
240
  }
168
241
  }
169
242
 
243
+
170
244
  export async function readMappedBufferCopy(stagingBuffer, options = {}) {
171
245
  const ownsStagingBuffer = options.ownsStagingBuffer !== false;
172
246
  let mapped = false;
@@ -193,11 +267,14 @@ export async function readBatchTokensFromStagingBuffers(options) {
193
267
  tokenCount,
194
268
  ownsTokensStaging = false,
195
269
  ownsStopStaging = false,
270
+ ownsFinitenessStaging = Boolean(finitenessStagingBuffer),
196
271
  ring = null,
272
+ cleanupRecorder = null,
197
273
  } = options;
198
274
  let tokensMapped = false;
199
275
  let stopMapped = false;
200
276
  let finitenessMapped = false;
277
+ let cleanupCompleted = false;
201
278
 
202
279
  try {
203
280
  const mapPromises = [tokensStagingBuffer.mapAsync(GPUMapMode.READ)];
@@ -207,10 +284,17 @@ export async function readBatchTokensFromStagingBuffers(options) {
207
284
  if (finitenessStagingBuffer) {
208
285
  mapPromises.push(finitenessStagingBuffer.mapAsync(GPUMapMode.READ));
209
286
  }
210
- await Promise.all(mapPromises);
211
- tokensMapped = true;
212
- stopMapped = Boolean(stopStagingBuffer);
213
- finitenessMapped = Boolean(finitenessStagingBuffer);
287
+ const mapResults = await Promise.allSettled(mapPromises);
288
+ tokensMapped = mapResults[0]?.status === 'fulfilled';
289
+ stopMapped = Boolean(stopStagingBuffer) && mapResults[1]?.status === 'fulfilled';
290
+ finitenessMapped = Boolean(finitenessStagingBuffer)
291
+ && mapResults[stopStagingBuffer ? 2 : 1]?.status === 'fulfilled';
292
+ const mapFailure = mapResults.find((result) => result.status === 'rejected');
293
+ if (mapFailure) {
294
+ throw mapFailure.reason;
295
+ }
296
+ cleanupRecorder?.completeDeferredCleanup();
297
+ cleanupCompleted = true;
214
298
 
215
299
  const tokens = Array.from(
216
300
  new Uint32Array(tokensStagingBuffer.getMappedRange()).subarray(0, tokenCount)
@@ -237,7 +321,10 @@ export async function readBatchTokensFromStagingBuffers(options) {
237
321
  if (stopMapped) {
238
322
  stopStagingBuffer.unmap();
239
323
  }
240
- if (finitenessStagingBuffer) {
324
+ if (!cleanupCompleted) {
325
+ cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
326
+ }
327
+ if (ownsFinitenessStaging) {
241
328
  finitenessStagingBuffer.destroy();
242
329
  }
243
330
  if (ownsTokensStaging) {
@@ -279,22 +366,53 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
279
366
  outputBuffer: decodeHiddenBuffer ?? undefined,
280
367
  transpose: state.embeddingTranspose,
281
368
  debugProbes: state.runtimeConfig.shared.debug.probes,
369
+ operatorDiagnostics: state.operatorDiagnostics,
282
370
  activationDtype,
283
371
  embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
372
+ executionPolicies: state.executionV1State?.policies ?? null,
284
373
  });
285
374
 
286
375
  let hiddenStates = embedTensor.buffer;
287
376
 
288
- for (let l = 0; l < config.numLayers; l++) {
289
- const prevStates = hiddenStates;
290
- hiddenStates = (await processLayer(l, hiddenStates, 1, false, context));
291
- state.decodeBuffers.swapPingPong();
292
- if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
293
- const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
294
- if (!isPreAllocated) {
295
- releaseBuffer(prevStates);
377
+ // Resolve pending PLE prefetch from previous decode step
378
+ let plePrefetchResult = null;
379
+ if (state.plePrefetchPending) {
380
+ plePrefetchResult = await state.plePrefetchPending;
381
+ state.plePrefetchPending = null;
382
+ }
383
+
384
+ const perLayerInputs = await preparePerLayerInputs([tokenId], embedTensor, context, {
385
+ numTokens: 1,
386
+ pleCache: state.pleCache ?? null,
387
+ prefetchedRow: plePrefetchResult,
388
+ });
389
+
390
+ try {
391
+ for (let l = 0; l < config.numLayers; l++) {
392
+ context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
393
+ const prevStates = hiddenStates;
394
+ hiddenStates = (await processLayer(l, hiddenStates, 1, false, context));
395
+ state.decodeBuffers.swapPingPong();
396
+ releasePerLayerInputBuffer(context.perLayerInputBuffer, null, context.decodeBuffers, state.pleCache ?? null);
397
+ if (perLayerInputs) {
398
+ perLayerInputs[l] = null;
399
+ }
400
+ context.perLayerInputBuffer = null;
401
+ if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
402
+ const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
403
+ if (!isPreAllocated) {
404
+ releaseBuffer(prevStates);
405
+ }
406
+ }
407
+ }
408
+ } finally {
409
+ context.perLayerInputBuffer = null;
410
+ if (perLayerInputs) {
411
+ for (const buffer of perLayerInputs) {
412
+ releasePerLayerInputBuffer(buffer, null, context.decodeBuffers, state.pleCache ?? null);
296
413
  }
297
414
  }
415
+ helpers.releaseSharedAttentionState?.(context.sharedAttentionState, null);
298
416
  }
299
417
 
300
418
  return { hiddenStates, decodeHiddenBuffer, decodeAltBuffer, debugCheckBuffer, context };
@@ -399,12 +517,27 @@ export async function decodeStep(state, currentIds, opts, helpers) {
399
517
  outputBuffer: decodeHiddenBuffer ?? undefined,
400
518
  transpose: state.embeddingTranspose,
401
519
  debugProbes: state.runtimeConfig.shared.debug.probes,
520
+ operatorDiagnostics: state.operatorDiagnostics,
402
521
  activationDtype,
403
522
  embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
523
+ executionPolicies: state.executionV1State?.policies ?? null,
404
524
  });
405
525
 
406
526
  let hiddenStates = embedTensor.buffer;
407
527
 
528
+ // Resolve pending PLE prefetch from previous decode step
529
+ let plePrefetchResult = null;
530
+ if (state.plePrefetchPending) {
531
+ plePrefetchResult = await state.plePrefetchPending;
532
+ state.plePrefetchPending = null;
533
+ }
534
+
535
+ const perLayerInputs = await preparePerLayerInputs([lastToken], embedTensor, context, {
536
+ numTokens: 1,
537
+ pleCache: state.pleCache ?? null,
538
+ prefetchedRow: plePrefetchResult,
539
+ });
540
+
408
541
  if (opts.debug && state.decodeStepCount === 1) {
409
542
  const validSize = config.hiddenSize * activationBytes;
410
543
  const embedData = await readBuffer(hiddenStates, validSize);
@@ -426,22 +559,48 @@ export async function decodeStep(state, currentIds, opts, helpers) {
426
559
  log.debug('Decode', `KV cache check: hasGPUCache=${hasGPUCache}, currentSeqLen=${context.currentSeqLen}`);
427
560
  }
428
561
 
429
- for (let l = 0; l < config.numLayers; l++) {
430
- const prevStates = hiddenStates;
431
- hiddenStates = (await processLayer(l, hiddenStates, numTokens, false, context));
432
-
433
- state.decodeBuffers.swapPingPong();
562
+ try {
563
+ for (let l = 0; l < config.numLayers; l++) {
564
+ context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
565
+ const prevStates = hiddenStates;
566
+ hiddenStates = (await processLayer(l, hiddenStates, numTokens, false, context));
567
+
568
+ state.decodeBuffers.swapPingPong();
569
+ releasePerLayerInputBuffer(
570
+ context.perLayerInputBuffer,
571
+ recorder,
572
+ context.decodeBuffers,
573
+ state.pleCache ?? null
574
+ );
575
+ if (perLayerInputs) {
576
+ perLayerInputs[l] = null;
577
+ }
578
+ context.perLayerInputBuffer = null;
434
579
 
435
- if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
436
- const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
437
- if (!isPreAllocated) {
438
- if (recorder) {
439
- recorder.trackTemporaryBuffer(prevStates);
440
- } else {
441
- releaseBuffer(prevStates);
580
+ if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
581
+ const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
582
+ if (!isPreAllocated) {
583
+ if (recorder) {
584
+ recorder.trackTemporaryBuffer(prevStates);
585
+ } else {
586
+ releaseBuffer(prevStates);
587
+ }
442
588
  }
443
589
  }
444
590
  }
591
+ } finally {
592
+ context.perLayerInputBuffer = null;
593
+ if (perLayerInputs) {
594
+ for (const buffer of perLayerInputs) {
595
+ releasePerLayerInputBuffer(
596
+ buffer,
597
+ recorder,
598
+ context.decodeBuffers,
599
+ state.pleCache ?? null
600
+ );
601
+ }
602
+ }
603
+ helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
445
604
  }
446
605
 
447
606
  const logitSoftcap = config.finalLogitSoftcapping === null
@@ -478,6 +637,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
478
637
  numTokens,
479
638
  helpers.getLogitsWeights(),
480
639
  helpers.getLogitsConfig(),
640
+ state.operatorDiagnostics,
481
641
  );
482
642
 
483
643
  const ringTokensBuffer = ringSlot?.tokens ?? null;
@@ -503,12 +663,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
503
663
 
504
664
  const ringStagingBuffer = ringSlot?.stagingTokens ?? null;
505
665
  const stagingSize = state.finitenessBuffer ? 20 : 4;
506
- const stagingBuffer = (ringStagingBuffer && !state.finitenessBuffer) ? ringStagingBuffer : device.createBuffer({
507
- label: 'sample_staging',
508
- size: stagingSize,
509
- usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
510
- });
511
- const ownsStagingBuffer = stagingBuffer !== ringStagingBuffer;
666
+ const stagingBuffer = ringStagingBuffer && ringStagingBuffer.size >= stagingSize
667
+ ? ringStagingBuffer
668
+ : getReusableSampleReadbackBuffer(state, device, stagingSize);
669
+ const ownsStagingBuffer = false;
512
670
  const ownsSampleOutputBuffer = !ringTokensBuffer || sampleOutputBuffer !== ringTokensBuffer;
513
671
 
514
672
  const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
@@ -518,25 +676,37 @@ export async function decodeStep(state, currentIds, opts, helpers) {
518
676
  encoder.copyBufferToBuffer(state.finitenessBuffer, 0, stagingBuffer, 4, 16);
519
677
  }
520
678
 
521
- const submitStart = performance.now();
522
- recorder.submit();
523
- const submitWaitMs = performance.now() - submitStart;
679
+ const readbackMode = executionPlan?.readbackMode;
680
+ const isOverlapped = readbackMode === 'overlapped';
681
+
682
+ // In overlapped mode, advance ring BEFORE submit so the GPU's next copy
683
+ // target is a fresh slot while we read the current one.
684
+ if (isOverlapped) {
685
+ ring?.advance();
686
+ }
524
687
 
525
688
  if (!allowReadback('pipeline.decode.sample')) {
526
689
  throw new Error('[Pipeline] GPU readback disabled for sampling');
527
690
  }
528
691
 
692
+ const submitStart = performance.now();
693
+ recorder.submit({ cleanup: 'deferred' });
694
+ const submitWaitMs = performance.now() - submitStart;
695
+
529
696
  const readbackStart = performance.now();
530
- const { nextToken, finitenessStatus } = await readSampledTokenFromStagingBuffer(stagingBuffer, {
697
+ const readbackResult = await readSampledTokenFromStagingBuffer(stagingBuffer, {
531
698
  ownsStagingBuffer,
532
699
  hasFinitenessBuffer: Boolean(state.finitenessBuffer),
533
- ring,
700
+ ring: isOverlapped ? null : ring,
701
+ cleanupRecorder: recorder,
534
702
  });
535
703
  const readbackWaitMs = performance.now() - readbackStart;
536
704
 
537
705
  state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
538
706
  state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
539
707
 
708
+ const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
709
+
540
710
  if (finitenessStatus.triggered) {
541
711
  releaseBuffer(logitsBuffer);
542
712
  if (ownsSampleOutputBuffer) releaseBuffer(sampleOutputBuffer);
@@ -544,13 +714,16 @@ export async function decodeStep(state, currentIds, opts, helpers) {
544
714
  throw new FinitenessError(`F16 bounds exceeded during generation${finitenessStatus.metadata}`);
545
715
  }
546
716
 
547
- log.debug('Decode', `Step ${state.decodeStepCount}: token=${nextToken} (vocabSize=${config.vocabSize})`);
717
+ log.debug('Decode', `Step ${state.decodeStepCount}: token=${fusedNextToken} (vocabSize=${config.vocabSize})`);
548
718
 
549
- const invalidToken = nextToken >= config.vocabSize
550
- || (padTokenId != null && nextToken === padTokenId)
551
- || (padTokenId == null && nextToken === 0);
719
+ const invalidToken = fusedNextToken >= config.vocabSize
720
+ || (padTokenId != null && fusedNextToken === padTokenId)
721
+ || (padTokenId == null && fusedNextToken === 0);
722
+ if (!invalidToken) {
723
+ schedulePlePrefetchForToken(state, fusedNextToken);
724
+ }
552
725
  if (invalidToken) {
553
- log.warn('Decode', `Suspicious token ${nextToken} (vocabSize=${config.vocabSize}, step=${state.decodeStepCount})`);
726
+ log.warn('Decode', `Suspicious token ${fusedNextToken} (vocabSize=${config.vocabSize}, step=${state.decodeStepCount})`);
554
727
  if (allowReadback('pipeline.decode.debug-logits')) {
555
728
  const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
556
729
  const logitSample = await readBuffer(logitsBuffer, Math.min(config.vocabSize * logitsBytes, 4096));
@@ -610,7 +783,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
610
783
  state.debugFlags,
611
784
  undefined,
612
785
  debugCheckBuffer,
613
- state.runtimeConfig.shared.debug.probes
786
+ state.runtimeConfig.shared.debug.probes,
787
+ null,
788
+ state.operatorDiagnostics
614
789
  );
615
790
  applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty);
616
791
  const fallbackToken = sample(fallbackLogits, {
@@ -620,6 +795,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
620
795
  padTokenId,
621
796
  seed: opts.seed,
622
797
  });
798
+ schedulePlePrefetchForToken(state, fallbackToken);
623
799
  if (!isPreAllocated) {
624
800
  releaseBuffer(hiddenStates);
625
801
  }
@@ -633,10 +809,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
633
809
 
634
810
  state.currentSeqLen++;
635
811
  const stepWallMs = performance.now() - stepWallStart;
636
- const gpuMs = state.stats.gpuTimeDecodeMs ?? 0;
637
812
  state.stats.singleTokenOrchestrationMs = (state.stats.singleTokenOrchestrationMs ?? 0)
638
813
  + Math.max(0, stepWallMs - submitWaitMs - readbackWaitMs);
639
- return nextToken;
814
+ return fusedNextToken;
640
815
  }
641
816
 
642
817
  await submitDecodeRecorderProfile(state, opts, recorder, ' (layers only)');
@@ -673,7 +848,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
673
848
  numTokens,
674
849
  helpers.getLogitsWeights(),
675
850
  helpers.getLogitsConfig(),
676
- state.debugFlags
851
+ state.debugFlags,
852
+ state.operatorDiagnostics
677
853
  );
678
854
  if (logitsResult) {
679
855
  const { logitsBuffer, vocabSize, logitsDtype } = logitsResult;
@@ -691,7 +867,8 @@ export async function decodeStep(state, currentIds, opts, helpers) {
691
867
  vocabSize,
692
868
  config.vocabSize,
693
869
  config,
694
- state.runtimeConfig.shared.debug.probes
870
+ state.runtimeConfig.shared.debug.probes,
871
+ state.operatorDiagnostics
695
872
  );
696
873
  const sampledLogits = extractLastPositionLogits(finalizedLogits, numTokens, config.vocabSize);
697
874
 
@@ -736,7 +913,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
736
913
  state.debugFlags,
737
914
  undefined,
738
915
  debugCheckBuffer,
739
- state.runtimeConfig.shared.debug.probes
916
+ state.runtimeConfig.shared.debug.probes,
917
+ null,
918
+ state.operatorDiagnostics
740
919
  );
741
920
 
742
921
  if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
@@ -794,7 +973,8 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
794
973
  numTokens,
795
974
  helpers.getLogitsWeights(),
796
975
  helpers.getLogitsConfig(),
797
- state.debugFlags
976
+ state.debugFlags,
977
+ state.operatorDiagnostics
798
978
  );
799
979
 
800
980
  if (logitsResult) {
@@ -811,7 +991,8 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
811
991
  rawVocabSize,
812
992
  config.vocabSize,
813
993
  config,
814
- state.runtimeConfig.shared.debug.probes
994
+ state.runtimeConfig.shared.debug.probes,
995
+ state.operatorDiagnostics
815
996
  );
816
997
  logits = extractLastPositionLogits(finalized, numTokens, config.vocabSize);
817
998
  }
@@ -827,7 +1008,9 @@ export async function decodeStepLogits(state, currentIds, opts, helpers) {
827
1008
  state.debugFlags,
828
1009
  undefined,
829
1010
  debugCheckBuffer,
830
- state.runtimeConfig.shared.debug.probes
1011
+ state.runtimeConfig.shared.debug.probes,
1012
+ null,
1013
+ state.operatorDiagnostics
831
1014
  );
832
1015
  logits = extractLastPositionLogits(rawLogits, numTokens, config.vocabSize);
833
1016
  }
@@ -940,6 +1123,61 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
940
1123
  };
941
1124
  }
942
1125
 
1126
+ async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, opts, helpers) {
1127
+ const config = state.modelConfig;
1128
+ const batchStart = performance.now();
1129
+ state.batchingStats.batchedForwardCalls += 1;
1130
+
1131
+ const stopTokenIds = config.stopTokenIds;
1132
+ const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
1133
+ const pleHiddenSize = Number(config.hiddenSizePerLayerInput ?? 0);
1134
+ const totalPerLayerHiddenSize = pleHiddenSize > 0
1135
+ ? config.numLayers * pleHiddenSize
1136
+ : 0;
1137
+ const pleWeights = state.weights.get('per_layer_inputs');
1138
+ const pleSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
1139
+ const generatedTokens = [];
1140
+ const rollingIds = Array.isArray(currentIds) ? currentIds.slice() : Array.from(currentIds ?? []);
1141
+ let gpuSubmissions = 0;
1142
+
1143
+ try {
1144
+ state.prefetchPleNextToken = true;
1145
+ if (
1146
+ totalPerLayerHiddenSize > 0
1147
+ && pleWeights?.embedTokensPerLayer
1148
+ && !state.plePrefetchPending
1149
+ && rollingIds.length > 0
1150
+ ) {
1151
+ state.plePrefetchPending = prefetchPerLayerRow(
1152
+ rollingIds[rollingIds.length - 1],
1153
+ pleWeights.embedTokensPerLayer,
1154
+ totalPerLayerHiddenSize,
1155
+ pleSession
1156
+ );
1157
+ }
1158
+
1159
+ for (let i = 0; i < N; i += 1) {
1160
+ const nextToken = await decodeStep(state, rollingIds, opts, helpers);
1161
+ gpuSubmissions += 1;
1162
+ generatedTokens.push(nextToken);
1163
+ rollingIds.push(nextToken);
1164
+
1165
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
1166
+ break;
1167
+ }
1168
+ }
1169
+
1170
+ return {
1171
+ tokens: generatedTokens,
1172
+ actualCount: generatedTokens.length,
1173
+ };
1174
+ } finally {
1175
+ state.prefetchPleNextToken = false;
1176
+ state.batchingStats.totalBatchedTimeMs += Math.max(0, performance.now() - batchStart);
1177
+ state.batchingStats.gpuSubmissions += gpuSubmissions;
1178
+ }
1179
+ }
1180
+
943
1181
  export async function generateNTokensGPU(state, startToken, N, currentIds, opts, helpers) {
944
1182
  const device = getDevice();
945
1183
  const config = state.modelConfig;
@@ -957,8 +1195,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
957
1195
  // GPU stop-flag checks are only useful when we read back every token.
958
1196
  // With deferred readback, we already scan sampled tokens on CPU to find the
959
1197
  // earliest stop token, so extra stop buffers/kernels are redundant overhead.
960
- const useGpuStopFlags = stopCheckMode === 'per-token' && readbackInterval <= 1;
961
- const effectiveStopCheckMode = useGpuStopFlags ? 'per-token' : 'batch';
1198
+ let useGpuStopFlags = stopCheckMode === 'per-token' && readbackInterval <= 1;
1199
+ let effectiveStopCheckMode = useGpuStopFlags ? 'per-token' : 'batch';
962
1200
  const batchStart = performance.now();
963
1201
 
964
1202
  state.batchingStats.batchedForwardCalls += 1;
@@ -978,6 +1216,30 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
978
1216
  throw new Error('[Pipeline] Batch size exceeds decode ring capacity.');
979
1217
  }
980
1218
 
1219
+ const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
1220
+ config,
1221
+ weights: state.weights,
1222
+ });
1223
+ const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: state.weights });
1224
+ const hotStartTokenIndex = pleHotVocabularyRuntime?.hotTokenIndexMap?.[startToken] ?? null;
1225
+ const canUseHotVocabularyBatchDecode = hasRangeBackedPerLayerInputs
1226
+ && pleHotVocabularyRuntime
1227
+ && hotStartTokenIndex != null
1228
+ && hotStartTokenIndex !== pleHotVocabularyRuntime.sentinelIndex;
1229
+ if (hasRangeBackedPerLayerInputs && !canUseHotVocabularyBatchDecode) {
1230
+ return generateNTokensGPUStepwiseRangeBackedPle(
1231
+ state,
1232
+ N,
1233
+ currentIds,
1234
+ opts,
1235
+ helpers
1236
+ );
1237
+ }
1238
+ if (canUseHotVocabularyBatchDecode) {
1239
+ useGpuStopFlags = true;
1240
+ effectiveStopCheckMode = 'per-token';
1241
+ }
1242
+
981
1243
  const stopTokenIds = config.stopTokenIds;
982
1244
  const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
983
1245
  const padTokenId = state.tokenizer?.getSpecialTokens?.()?.pad ?? null;
@@ -991,7 +1253,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
991
1253
  if (eosTokenId == null) {
992
1254
  throw new Error('[Pipeline] Missing EOS token. Ensure tokenizer or manifest provides stop tokens.');
993
1255
  }
994
- const maxTokens = executionPlan?.maxTokens ?? opts.maxTokens ?? state.runtimeConfig.inference.batching.maxTokens;
1256
+ const maxTokens = executionPlan?.maxTokens
1257
+ ?? opts.maxTokens
1258
+ ?? state.runtimeConfig.inference.generation.maxTokens;
995
1259
  const maxSeqLen = state.currentSeqLen + maxTokens;
996
1260
 
997
1261
  const recordStart = performance.now();
@@ -1025,6 +1289,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1025
1289
  })
1026
1290
  : null;
1027
1291
  const ownsStopBuffer = useGpuStopFlags && !ringSlot?.stop;
1292
+ const pleInputTokensBuffer = canUseHotVocabularyBatchDecode
1293
+ ? device.createBuffer({
1294
+ size: (tokenCapacity + 1) * 4,
1295
+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1296
+ label: 'ple_hot_input_tokens',
1297
+ })
1298
+ : null;
1028
1299
 
1029
1300
  const tokensStagingBuffer = ringSlot?.stagingTokens ?? device.createBuffer({
1030
1301
  size: N * 4,
@@ -1039,7 +1310,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1039
1310
  })
1040
1311
  : null;
1041
1312
  const ownsStopStaging = useGpuStopFlags && !ringSlot?.stagingStop;
1042
- let finitenessStagingBuffer = null;
1313
+ const finitenessStagingBuffer = state.finitenessBuffer
1314
+ ? ringSlot?.stagingFiniteness ?? device.createBuffer({
1315
+ size: 16,
1316
+ usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
1317
+ })
1318
+ : null;
1319
+ const ownsFinitenessStaging = Boolean(state.finitenessBuffer) && !ringSlot?.stagingFiniteness;
1043
1320
  let readbackCleanupDelegated = false;
1044
1321
  let repHistoryBuffer = null;
1045
1322
  let repHistoryCount = 0;
@@ -1049,6 +1326,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1049
1326
  }
1050
1327
 
1051
1328
  device.queue.writeBuffer(tokensBuffer, 0, new Uint32Array([startToken]));
1329
+ if (pleInputTokensBuffer) {
1330
+ device.queue.writeBuffer(pleInputTokensBuffer, 0, new Uint32Array([hotStartTokenIndex]));
1331
+ }
1052
1332
  if (stopBuffer) {
1053
1333
  const stopElements = stopBuffer.size / 4;
1054
1334
  const zeroStopData = ringSlot?.zeroStopData;
@@ -1085,10 +1365,15 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1085
1365
  device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
1086
1366
  }
1087
1367
 
1368
+ // Hoist loop-invariant values to avoid repeated rule lookups and allocations.
1369
+ const embeddingDtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype });
1370
+ const debugProbes = state.runtimeConfig.shared.debug.probes;
1371
+ const currentTokenIdsArray = [startToken];
1372
+
1088
1373
  for (let i = 0; i < N; i++) {
1089
1374
  const currentPos = state.currentSeqLen + i;
1090
1375
  context.currentSeqLen = currentPos;
1091
- context.currentTokenIds = [startToken];
1376
+ context.currentTokenIds = currentTokenIdsArray;
1092
1377
  context.decodeBuffers?.resetPingPong();
1093
1378
 
1094
1379
  const hiddenTensor = await embed(tokensBuffer, embedBuffer, {
@@ -1097,24 +1382,59 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1097
1382
  scaleEmbeddings: config.scaleEmbeddings,
1098
1383
  recorder,
1099
1384
  transpose: state.embeddingTranspose,
1100
- debugProbes: state.runtimeConfig.shared.debug.probes,
1385
+ debugProbes,
1386
+ operatorDiagnostics: state.operatorDiagnostics,
1101
1387
  activationDtype,
1102
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
1388
+ embeddingDtype,
1389
+ executionPolicies: state.executionV1State?.policies ?? null,
1103
1390
  numTokens: 1,
1104
1391
  indexOffset: i,
1105
1392
  });
1106
1393
 
1107
1394
  let hiddenStatesBuffer = hiddenTensor.buffer;
1108
- for (let l = 0; l < config.numLayers; l++) {
1109
- const prevStates = hiddenStatesBuffer;
1110
- hiddenStatesBuffer = (await processLayer(l, hiddenStatesBuffer, 1, false, context));
1111
- context.decodeBuffers?.swapPingPong();
1112
- if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStatesBuffer) {
1113
- const ownsBuffer = context.decodeBuffers?.ownsBuffer(prevStates);
1114
- if (!ownsBuffer) {
1115
- recorder.trackTemporaryBuffer(prevStates);
1395
+ const perLayerInputs = await preparePerLayerInputs(tokensBuffer, hiddenTensor, context, {
1396
+ numTokens: 1,
1397
+ indexOffset: i,
1398
+ perLayerTokenIds: pleInputTokensBuffer,
1399
+ perLayerIndexOffset: i,
1400
+ pleCache: state.pleCache ?? null,
1401
+ });
1402
+ try {
1403
+ for (let l = 0; l < config.numLayers; l++) {
1404
+ context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
1405
+ const prevStates = hiddenStatesBuffer;
1406
+ hiddenStatesBuffer = (await processLayer(l, hiddenStatesBuffer, 1, false, context));
1407
+ context.decodeBuffers?.swapPingPong();
1408
+ releasePerLayerInputBuffer(
1409
+ context.perLayerInputBuffer,
1410
+ recorder,
1411
+ context.decodeBuffers,
1412
+ state.pleCache ?? null
1413
+ );
1414
+ if (perLayerInputs) {
1415
+ perLayerInputs[l] = null;
1416
+ }
1417
+ context.perLayerInputBuffer = null;
1418
+ if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStatesBuffer) {
1419
+ const ownsBuffer = context.decodeBuffers?.ownsBuffer(prevStates);
1420
+ if (!ownsBuffer) {
1421
+ recorder.trackTemporaryBuffer(prevStates);
1422
+ }
1423
+ }
1424
+ }
1425
+ } finally {
1426
+ context.perLayerInputBuffer = null;
1427
+ if (perLayerInputs) {
1428
+ for (const buffer of perLayerInputs) {
1429
+ releasePerLayerInputBuffer(
1430
+ buffer,
1431
+ recorder,
1432
+ context.decodeBuffers,
1433
+ state.pleCache ?? null
1434
+ );
1116
1435
  }
1117
1436
  }
1437
+ helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
1118
1438
  }
1119
1439
 
1120
1440
  const logits = await recordLogitsGPU(
@@ -1122,7 +1442,8 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1122
1442
  hiddenStatesBuffer,
1123
1443
  1,
1124
1444
  helpers.getLogitsWeights(),
1125
- helpers.getLogitsConfig()
1445
+ helpers.getLogitsConfig(),
1446
+ state.operatorDiagnostics
1126
1447
  );
1127
1448
  const { logitsBuffer, vocabSize, logitsDtype } = logits;
1128
1449
 
@@ -1160,7 +1481,19 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1160
1481
  });
1161
1482
  }
1162
1483
 
1163
- const stopCheck = useGpuStopFlags
1484
+ const stopCheck = canUseHotVocabularyBatchDecode
1485
+ ? recordCheckHotVocabStop(recorder, {
1486
+ sampledTokenBuffer: tokensBuffer,
1487
+ nextInputTokenBuffer: pleInputTokensBuffer,
1488
+ hotTokenIndexMapBuffer: pleHotVocabularyRuntime.hotTokenIndexMapBuffer,
1489
+ hotTokenSentinel: pleHotVocabularyRuntime.sentinelIndex,
1490
+ shouldStopBuffer: stopBuffer,
1491
+ tokenIndex: outputIndex,
1492
+ eosTokenId,
1493
+ maxTokens: maxSeqLen,
1494
+ currentPos,
1495
+ })
1496
+ : useGpuStopFlags
1164
1497
  ? recordCheckStop(recorder, {
1165
1498
  sampledTokenBuffer: tokensBuffer,
1166
1499
  shouldStopBuffer: stopBuffer,
@@ -1191,20 +1524,16 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1191
1524
  encoder.copyBufferToBuffer(stopBuffer, 4, stopStagingBuffer, 0, N * 4);
1192
1525
  }
1193
1526
 
1194
- if (state.finitenessBuffer) {
1195
- finitenessStagingBuffer = device.createBuffer({
1196
- size: 16,
1197
- usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ
1198
- });
1527
+ if (state.finitenessBuffer && finitenessStagingBuffer) {
1199
1528
  encoder.copyBufferToBuffer(state.finitenessBuffer, 0, finitenessStagingBuffer, 0, 16);
1200
1529
  }
1201
1530
 
1202
- recorder.submit();
1203
-
1204
1531
  if (!allowReadback('pipeline.decode.sample')) {
1205
1532
  throw new Error('[Pipeline] GPU readback disabled for sampling');
1206
1533
  }
1207
1534
 
1535
+ recorder.submit({ cleanup: 'deferred' });
1536
+
1208
1537
  const readbackStart = performance.now();
1209
1538
  readbackCleanupDelegated = true;
1210
1539
  const readback = await readBatchTokensFromStagingBuffers({
@@ -1214,7 +1543,9 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1214
1543
  tokenCount: N,
1215
1544
  ownsTokensStaging,
1216
1545
  ownsStopStaging,
1546
+ ownsFinitenessStaging,
1217
1547
  ring,
1548
+ cleanupRecorder: recorder,
1218
1549
  });
1219
1550
  const readbackWaitMs = performance.now() - readbackStart;
1220
1551
  state.stats.decodeReadbackWaitMs = (state.stats.decodeReadbackWaitMs ?? 0) + readbackWaitMs;
@@ -1280,7 +1611,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1280
1611
  state.batchingStats.gpuSubmissions += 1;
1281
1612
 
1282
1613
  if (!readbackCleanupDelegated) {
1283
- if (finitenessStagingBuffer) {
1614
+ if (ownsFinitenessStaging) {
1284
1615
  finitenessStagingBuffer.destroy();
1285
1616
  }
1286
1617
  if (ownsTokensStaging) tokensStagingBuffer.destroy();
@@ -1289,6 +1620,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1289
1620
  }
1290
1621
  if (ownsTokensBuffer) tokensBuffer.destroy();
1291
1622
  if (ownsStopBuffer) stopBuffer?.destroy();
1623
+ pleInputTokensBuffer?.destroy();
1292
1624
  if (repHistoryBuffer) repHistoryBuffer.destroy();
1293
1625
  }
1294
1626
  }