doppler-gpu 0.4.15 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1063) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -213
  4. package/models/catalog.json +241 -33
  5. package/package.json +122 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +101 -1349
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +4 -8
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +64 -1
  28. package/src/client/doppler-api.js +42 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.d.ts +122 -0
  41. package/src/client/inspection.js +343 -0
  42. package/src/client/model-host/index.d.ts +140 -0
  43. package/src/client/model-host/index.js +548 -0
  44. package/src/client/model-host/model-evidence.js +255 -0
  45. package/src/client/model-host/model-session.d.ts +322 -0
  46. package/src/client/model-host/model-session.js +593 -0
  47. package/src/client/provider.d.ts +18 -2
  48. package/src/client/provider.js +26 -3
  49. package/src/client/receipt.js +30 -72
  50. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  51. package/src/client/runtime/capsule-acquisition.js +103 -0
  52. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  53. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  54. package/src/client/runtime/capsule-artifact-source.js +62 -0
  55. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  56. package/src/client/runtime/capsule-embedding.js +76 -0
  57. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  58. package/src/client/runtime/capsule-forecast-program.js +85 -0
  59. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  60. package/src/client/runtime/capsule-forecast.js +31 -0
  61. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  62. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  63. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  64. package/src/client/runtime/capsule-operation-executor.js +100 -0
  65. package/src/client/runtime/capsule-program-adapter.js +151 -0
  66. package/src/client/runtime/capsule-release-authorization.js +25 -0
  67. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  68. package/src/client/runtime/capsule-rerank.js +105 -0
  69. package/src/client/runtime/capsule-session-execution.js +85 -0
  70. package/src/client/runtime/command-executor.d.ts +15 -0
  71. package/src/client/runtime/command-executor.js +128 -0
  72. package/src/client/runtime/composition-root.d.ts +74 -0
  73. package/src/client/runtime/composition-root.js +344 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  75. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  76. package/src/client/runtime/index.d.ts +1 -63
  77. package/src/client/runtime/index.js +1 -261
  78. package/src/client/runtime/lora.d.ts +14 -0
  79. package/src/client/runtime/lora.js +19 -1
  80. package/src/client/runtime/model-manager.d.ts +0 -4
  81. package/src/client/runtime/model-manager.js +12 -28
  82. package/src/client/runtime/model-rerank.js +30 -0
  83. package/src/client/runtime/model-session.d.ts +1 -174
  84. package/src/client/runtime/model-source.d.ts +19 -1
  85. package/src/client/runtime/model-source.js +46 -17
  86. package/src/client/runtime/node-quickstart-cache.js +1 -1
  87. package/src/client/runtime/resolution-policy.d.ts +20 -0
  88. package/src/client/runtime/resolution-policy.js +84 -0
  89. package/src/client/runtime/resource-binder.d.ts +24 -0
  90. package/src/client/runtime/resource-binder.js +135 -0
  91. package/src/client/runtime/scoped-session.d.ts +126 -0
  92. package/src/client/runtime/scoped-session.js +276 -0
  93. package/src/client/runtime/session-controller.d.ts +30 -0
  94. package/src/client/runtime/session-controller.js +170 -0
  95. package/src/client/runtime/source-runtime.js +1 -1
  96. package/src/client/runtime/target-selector.d.ts +9 -0
  97. package/src/client/runtime/target-selector.js +2 -0
  98. package/src/client/runtime/types.d.ts +1 -1
  99. package/src/client/runtime/types.js +0 -1
  100. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  101. package/src/client/wrap-pipeline-handle.js +3 -9
  102. package/src/config/backward-registry-loader.js +1 -1
  103. package/src/config/capsule-adapter-policy.js +30 -0
  104. package/src/config/capsule-adapters.d.ts +21 -0
  105. package/src/config/capsule-adapters.js +40 -0
  106. package/src/config/capsule-adapters.json +9 -0
  107. package/src/config/capsule-loading.d.ts +2 -0
  108. package/src/config/capsule-loading.js +14 -0
  109. package/src/config/capsule-loading.schema.json +11 -0
  110. package/src/config/capsule-operation.d.ts +19 -0
  111. package/src/config/capsule-operation.js +58 -0
  112. package/src/config/capsule-operations.json +18 -0
  113. package/src/config/capsule-release-contract.d.ts +66 -0
  114. package/src/config/capsule-release-contract.js +197 -0
  115. package/src/config/capsule-release-events.d.ts +53 -0
  116. package/src/config/capsule-release-events.js +172 -0
  117. package/src/config/capsule-sequence-receipt.js +19 -0
  118. package/src/config/capsule-serve.d.ts +9 -0
  119. package/src/config/capsule-serve.js +28 -0
  120. package/src/config/capsule-serve.schema.json +14 -0
  121. package/src/config/capsule-signature.d.ts +6 -0
  122. package/src/config/capsule-signature.js +50 -0
  123. package/src/config/capsule-v2.d.ts +70 -0
  124. package/src/config/capsule-v2.js +431 -0
  125. package/src/config/capsule-v3.d.ts +19 -0
  126. package/src/config/capsule-v3.js +72 -0
  127. package/src/config/capsule.d.ts +22 -0
  128. package/src/config/capsule.js +52 -0
  129. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  130. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  132. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  136. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  137. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  144. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  147. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  150. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  151. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  152. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  155. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  157. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  158. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  160. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  161. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  162. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  163. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  164. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  165. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  166. package/src/config/dtype-transition-contract.js +40 -0
  167. package/src/config/embedding-contract.js +26 -0
  168. package/src/config/embedding-reference.js +85 -0
  169. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  170. package/src/config/evidence/source-intake-policy.json +122 -0
  171. package/src/config/execution-contract-check.d.ts +16 -73
  172. package/src/config/execution-contract-check.js +7 -429
  173. package/src/config/forecast-manifest.js +56 -0
  174. package/src/config/heterogeneous-attention-contract.js +46 -0
  175. package/src/config/initial-execution-identity.d.ts +58 -0
  176. package/src/config/initial-execution-identity.js +331 -0
  177. package/src/config/inspection/observation-policies.json +80 -0
  178. package/src/config/kernel-path-loader.d.ts +10 -0
  179. package/src/config/kernel-path-loader.js +23 -8
  180. package/src/config/kernel-registry-contract.d.ts +25 -0
  181. package/src/config/kernel-registry-contract.js +46 -0
  182. package/src/config/kernels/kernel-ref-digests.js +50 -2
  183. package/src/config/kernels/registry.js +1 -1
  184. package/src/config/kernels/registry.json +13592 -11377
  185. package/src/config/lora-layouts.d.ts +10 -0
  186. package/src/config/lora-layouts.js +17 -0
  187. package/src/config/lora-layouts.json +9 -0
  188. package/src/config/manifest-inference-validation.js +575 -0
  189. package/src/config/merge/execution-patches.js +12 -0
  190. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  191. package/src/config/merge/value-selection.js +38 -0
  192. package/src/config/merge.js +43 -11
  193. package/src/config/model-ir-v2.d.ts +57 -0
  194. package/src/config/model-ir-v2.js +258 -0
  195. package/src/config/model-ir.d.ts +49 -0
  196. package/src/config/model-ir.js +180 -0
  197. package/src/config/param-validator.js +22 -37
  198. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  199. package/src/config/platforms/loader.js +2 -1
  200. package/src/config/production-release-evidence.d.ts +23 -0
  201. package/src/config/production-release-evidence.js +423 -0
  202. package/src/config/production-release.js +321 -0
  203. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  204. package/src/config/required-inference-fields-contract-check.js +2 -2
  205. package/src/config/rerank-reference.d.ts +28 -0
  206. package/src/config/rerank-reference.js +89 -0
  207. package/src/config/revocation-policy.d.ts +65 -0
  208. package/src/config/revocation-policy.js +290 -0
  209. package/src/config/revocation-registry.json +11 -0
  210. package/src/config/revocation-updates.d.ts +68 -0
  211. package/src/config/revocation-updates.js +440 -0
  212. package/src/config/runtime/compute/f16-activations.json +1 -0
  213. package/src/config/runtime/compute/f16-batched.json +1 -0
  214. package/src/config/runtime/default.json +1 -0
  215. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  216. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  217. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  218. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  219. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  220. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  221. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  222. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  223. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  224. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  225. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  226. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  227. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  228. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  229. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  230. package/src/config/runtime/model/gemma2-debug.json +1 -0
  231. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  232. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  233. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  234. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  235. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  236. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  237. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  238. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  239. package/src/config/runtime/profiles/default.json +1 -0
  240. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  241. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  242. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  243. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  244. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  245. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +2 -6
  246. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  247. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  248. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  249. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  250. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  251. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  252. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  253. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  254. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  255. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  256. package/src/config/runtime/profiles/low-memory.json +1 -0
  257. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  258. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  259. package/src/config/runtime/profiles/production.json +1 -3
  260. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  261. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  262. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  263. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  264. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  265. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  266. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  267. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  268. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  269. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  270. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  271. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  272. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  273. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  274. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  275. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  276. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  277. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +2 -6
  278. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  279. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  280. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  281. package/src/config/runtime/profiles/simulation.json +1 -3
  282. package/src/config/runtime/profiles/throughput.json +1 -3
  283. package/src/config/runtime/profiles/trace-layers.json +1 -3
  284. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  285. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  286. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  287. package/src/config/runtime/profiles/turboquant.json +1 -0
  288. package/src/config/runtime/profiles/vector-stability.json +1 -3
  289. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  290. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  291. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  292. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  293. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  294. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  295. package/src/config/runtime-merge.js +1 -1
  296. package/src/config/runtime.js +1 -1
  297. package/src/config/schema/benchmark.schema.d.ts +7 -0
  298. package/src/config/schema/benchmark.schema.js +5 -0
  299. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  300. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  301. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  302. package/src/config/schema/conversion-report.schema.js +29 -0
  303. package/src/config/schema/converter.schema.d.ts +1 -0
  304. package/src/config/schema/debug.schema.d.ts +1 -0
  305. package/src/config/schema/doppler.schema.js +5 -5
  306. package/src/config/schema/emulation.schema.js +1 -1
  307. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  308. package/src/config/schema/execution-v1.schema.js +19 -0
  309. package/src/config/schema/harness.schema.d.ts +13 -0
  310. package/src/config/schema/harness.schema.js +6 -0
  311. package/src/config/schema/index.d.ts +8 -16
  312. package/src/config/schema/index.js +5 -8
  313. package/src/config/schema/inference.schema.d.ts +21 -2
  314. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  315. package/src/config/schema/kernel-thresholds.schema.js +1 -13
  316. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  317. package/src/config/schema/kernel-warmup.schema.js +0 -1
  318. package/src/config/schema/manifest.schema.d.ts +26 -7
  319. package/src/config/schema/manifest.schema.js +10 -1
  320. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  321. package/src/config/schema/program-bundle.schema.js +145 -10
  322. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  323. package/src/config/schema/runtime-profile.schema.js +92 -0
  324. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  325. package/src/config/schema/shared-runtime.schema.js +0 -2
  326. package/src/config/schema/tooling.schema.d.ts +8 -5
  327. package/src/config/schema/tooling.schema.js +3 -2
  328. package/src/config/sequence-reference.d.ts +26 -0
  329. package/src/config/sequence-reference.js +100 -0
  330. package/src/config/target-plan.d.ts +108 -0
  331. package/src/config/target-plan.js +411 -0
  332. package/src/config/training-defaults.js +1 -1
  333. package/src/config/transforms/capability-transform-resolver.js +4 -105
  334. package/src/config/transforms/execution-graph/dtype.js +591 -0
  335. package/src/config/transforms/execution-graph/fusion.js +436 -0
  336. package/src/config/transforms/execution-graph/validation.js +414 -0
  337. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  338. package/src/converter/artifact-identity.js +475 -0
  339. package/src/converter/conversion-plan.js +1 -1
  340. package/src/converter/core.js +94 -2096
  341. package/src/converter/forge-candidate-evaluation.js +271 -0
  342. package/src/converter/forge-qualification.js +162 -0
  343. package/src/converter/forge-stages.js +798 -0
  344. package/src/converter/manifest-builder.js +683 -0
  345. package/src/converter/quantization-info.js +4 -1
  346. package/src/converter/quantizer.js +58 -9
  347. package/src/converter/shard-packer.d.ts +1 -1
  348. package/src/converter/shard-packer.js +2 -2
  349. package/src/converter/tensor-transform.js +640 -0
  350. package/src/converter/tokenizer-contract.js +243 -0
  351. package/src/converter/tokenizer-manifest.js +243 -0
  352. package/src/debug/history.js +0 -8
  353. package/src/debug/index.js +0 -10
  354. package/src/debug/tensor.d.ts +2 -0
  355. package/src/debug/tensor.js +21 -2
  356. package/src/debug/trace.js +0 -7
  357. package/src/experimental/adapters/adapter-manager.js +2 -0
  358. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  359. package/src/experimental/adapters/adapter-manifest.js +11 -0
  360. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  361. package/src/experimental/adapters/artifact-contract.js +5 -2
  362. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  363. package/src/experimental/adapters/lora-loader.js +110 -48
  364. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  365. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  366. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  367. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  368. package/src/experimental/browser/browser-converter.js +11 -11
  369. package/src/experimental/diffusion/index.d.ts +1 -1
  370. package/src/experimental/diffusion/index.js +1 -1
  371. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  372. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  373. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  374. package/src/experimental/diffusion/vae.js +596 -0
  375. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  376. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  377. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  378. package/src/experimental/distribution/shard-delivery.js +4 -1709
  379. package/src/experimental/energy/index.d.ts +2 -2
  380. package/src/experimental/energy/index.js +2 -2
  381. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  382. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  383. package/src/experimental/logits/cpu-output.d.ts +27 -0
  384. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  385. package/src/experimental/logits/cpu.d.ts +87 -0
  386. package/src/experimental/logits/cpu.js +95 -0
  387. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  388. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  389. package/src/experimental/orchestration/index.d.ts +1 -1
  390. package/src/experimental/orchestration/index.js +1 -1
  391. package/src/experimental/training/artifacts.js +2 -2
  392. package/src/experimental/training/checkpoint.js +2 -2
  393. package/src/experimental/training/distillation/artifacts.js +1 -1
  394. package/src/experimental/training/distillation/dataset.js +2 -2
  395. package/src/experimental/training/distillation/eval.js +1 -1
  396. package/src/experimental/training/distillation/runtime.js +2 -2
  397. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  398. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  399. package/src/experimental/training/distillation/student-fixture.js +2 -469
  400. package/src/experimental/training/lora/checkpoint.js +746 -0
  401. package/src/experimental/training/lora/execution.js +550 -0
  402. package/src/experimental/training/lora/plan.js +387 -0
  403. package/src/experimental/training/lora/recovery.js +418 -0
  404. package/src/experimental/training/lora-pipeline.js +7 -1949
  405. package/src/experimental/training/operator-artifacts.js +2 -2
  406. package/src/experimental/training/operator-command.js +4 -4
  407. package/src/experimental/training/runner/evaluation.js +62 -0
  408. package/src/experimental/training/runner/lifecycle.js +257 -0
  409. package/src/experimental/training/runner.js +5 -268
  410. package/src/experimental/training/suite/execution.js +879 -0
  411. package/src/experimental/training/suite/plan.js +855 -0
  412. package/src/experimental/training/suite.js +5 -1659
  413. package/src/experimental/training/workloads.d.ts +5 -5
  414. package/src/experimental/training/workloads.js +6 -6
  415. package/src/experimental/turboquant/codebook.js +470 -0
  416. package/src/formats/quantization-constants.d.ts +11 -0
  417. package/src/formats/quantization-constants.js +11 -0
  418. package/src/formats/rdrr/integrity.js +1 -1
  419. package/src/formats/rdrr/lowerings.js +0 -29
  420. package/src/formats/rdrr/merkle.js +1 -1
  421. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  422. package/src/formats/rdrr/validation.js +3 -3
  423. package/src/gpu/command-recorder.d.ts +15 -1
  424. package/src/gpu/command-recorder.js +63 -2
  425. package/src/gpu/constant-buffer.js +29 -0
  426. package/src/gpu/device-state.js +36 -0
  427. package/src/gpu/device.js +6 -41
  428. package/src/gpu/kernel-runtime.js +3 -10
  429. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  430. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  431. package/src/gpu/kernels/attention/executors.js +960 -0
  432. package/src/gpu/kernels/attention/plan.js +472 -0
  433. package/src/gpu/kernels/attention.d.ts +4 -0
  434. package/src/gpu/kernels/attention.js +145 -1545
  435. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  436. package/src/gpu/kernels/backward/adam.js +2 -1
  437. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  438. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  439. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  440. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  441. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  442. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  443. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  444. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  445. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  446. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  447. package/src/gpu/kernels/backward/executor.js +158 -0
  448. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  449. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  450. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  451. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  452. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  453. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  454. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  455. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  456. package/src/gpu/kernels/backward/matmul.js +133 -0
  457. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  458. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  459. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  460. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  461. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  462. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  463. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  464. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  465. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  466. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  467. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  468. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  469. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  470. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  471. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  472. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  473. package/src/gpu/kernels/backward/utils.js +2 -392
  474. package/src/gpu/kernels/bind-group-validation.js +16 -0
  475. package/src/gpu/kernels/cast.js +2 -1
  476. package/src/gpu/kernels/check-finiteness.js +13 -49
  477. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  478. package/src/gpu/kernels/check-stop.js +14 -33
  479. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  480. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  481. package/src/gpu/kernels/check_stop.wgsl +20 -0
  482. package/src/gpu/kernels/clamp.js +1 -1
  483. package/src/gpu/kernels/conv2d.js +1 -1
  484. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  485. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  486. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  487. package/src/gpu/kernels/dequant.js +7 -2
  488. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  489. package/src/gpu/kernels/dispatch.js +12 -1
  490. package/src/gpu/kernels/embedding-pool.js +72 -0
  491. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  492. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  493. package/src/gpu/kernels/energy.js +2 -1
  494. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  495. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  496. package/src/gpu/kernels/fused_ffn.js +10 -220
  497. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  498. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  499. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  500. package/src/gpu/kernels/gated-short-conv.js +11 -69
  501. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  502. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  503. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  504. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  505. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  506. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  507. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  508. package/src/gpu/kernels/gather.d.ts +1 -1
  509. package/src/gpu/kernels/gather.js +30 -6
  510. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  511. package/src/gpu/kernels/gelu.js +1 -1
  512. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  513. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  514. package/src/gpu/kernels/groupnorm.js +1 -1
  515. package/src/gpu/kernels/index.d.ts +38 -7
  516. package/src/gpu/kernels/index.js +18 -6
  517. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  518. package/src/gpu/kernels/kernel-base.js +1 -1
  519. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  520. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  521. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  522. package/src/gpu/kernels/kernel-configs.js +26 -54
  523. package/src/gpu/kernels/kernel-execution.js +141 -0
  524. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  525. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  526. package/src/gpu/kernels/kv-cache-write.js +4 -4
  527. package/src/gpu/kernels/kv-quantize.js +4 -1
  528. package/src/gpu/kernels/l2-normalize.js +36 -0
  529. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  530. package/src/gpu/kernels/layernorm.d.ts +4 -4
  531. package/src/gpu/kernels/layernorm.js +44 -7
  532. package/src/gpu/kernels/linear-attention-core.js +29 -449
  533. package/src/gpu/kernels/linear_attention.js +1 -1
  534. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  535. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  536. package/src/gpu/kernels/logit-finalize.js +103 -0
  537. package/src/gpu/kernels/logit-merge.js +13 -125
  538. package/src/gpu/kernels/logit-scatter.js +40 -0
  539. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  540. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  541. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  542. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  543. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  544. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  545. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  546. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  547. package/src/gpu/kernels/matmul-selection.js +3 -2
  548. package/src/gpu/kernels/matmul.js +4 -15
  549. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  550. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  551. package/src/gpu/kernels/modulate.js +1 -1
  552. package/src/gpu/kernels/moe.js +3 -1
  553. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  554. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  555. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  556. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  557. package/src/gpu/kernels/patch_context.wgsl +29 -0
  558. package/src/gpu/kernels/pipeline-cache.js +3 -1
  559. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  560. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  561. package/src/gpu/kernels/relu.js +1 -1
  562. package/src/gpu/kernels/repeat_channels.js +1 -1
  563. package/src/gpu/kernels/residual.js +2 -2
  564. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  565. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  566. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  567. package/src/gpu/kernels/rmsnorm.js +1 -1
  568. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  569. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  570. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  571. package/src/gpu/kernels/rope-precompute.js +160 -0
  572. package/src/gpu/kernels/rope-qk.js +1 -1
  573. package/src/gpu/kernels/rope.js +2 -2
  574. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  575. package/src/gpu/kernels/sample.js +5 -1
  576. package/src/gpu/kernels/scale.js +1 -1
  577. package/src/gpu/kernels/shader-cache.js +8 -4
  578. package/src/gpu/kernels/shader-source-scope.js +68 -0
  579. package/src/gpu/kernels/silu.js +2 -1
  580. package/src/gpu/kernels/soft-embedding.js +3 -1
  581. package/src/gpu/kernels/softmax.js +4 -2
  582. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  583. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  584. package/src/gpu/kernels/split_qg.js +1 -1
  585. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  586. package/src/gpu/kernels/split_qkv.js +18 -3
  587. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  588. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  589. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  590. package/src/gpu/kernels/transpose.js +1 -1
  591. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  592. package/src/gpu/kernels/uniform-utils.js +7 -2
  593. package/src/gpu/kernels/upsample2d.js +1 -1
  594. package/src/gpu/kernels/utils.js +9 -332
  595. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  596. package/src/gpu/kernels/vision-average-pool.js +68 -0
  597. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  598. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  599. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  600. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  601. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  602. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  603. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  604. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  605. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  606. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  607. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  608. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  609. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  610. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  611. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  612. package/src/gpu/readback-buffer.js +20 -0
  613. package/src/gpu/submit-probe.js +2 -26
  614. package/src/gpu/weight-buffer.d.ts +1 -0
  615. package/src/gpu/weight-buffer.js +7 -0
  616. package/src/index.d.ts +23 -1
  617. package/src/index.js +8 -1
  618. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  619. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  620. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  621. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  622. package/src/inference/browser-harness/inference-execution.js +543 -0
  623. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +18 -145
  624. package/src/inference/browser-harness/model-resolution.js +137 -0
  625. package/src/inference/browser-harness/report.js +338 -0
  626. package/src/inference/browser-harness/request.js +263 -0
  627. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  628. package/src/inference/browser-harness/runtime-config.js +255 -0
  629. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  630. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  631. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  632. package/src/inference/browser-harness/runtime-report.js +117 -0
  633. package/src/inference/browser-harness/suite-summary.js +39 -0
  634. package/src/inference/browser-harness/text-evidence.js +694 -0
  635. package/src/inference/browser-harness/text-execution.js +757 -0
  636. package/src/inference/browser-harness/text-input.js +725 -0
  637. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  638. package/src/inference/browser-harness-contract-helpers.js +1 -1
  639. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  640. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  641. package/src/inference/browser-harness-runtime-helpers.js +3 -408
  642. package/src/inference/browser-harness-text-helpers.js +31 -2196
  643. package/src/inference/browser-harness.d.ts +19 -2
  644. package/src/inference/browser-harness.js +46 -1873
  645. package/src/inference/kv-cache/base.js +67 -381
  646. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  647. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  648. package/src/inference/kv-cache/quantized.d.ts +2 -2
  649. package/src/inference/kv-cache/quantized.js +4 -8
  650. package/src/inference/kv-cache/update.js +309 -0
  651. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  652. package/src/inference/model-checkpoint-evidence.js +264 -0
  653. package/src/inference/moe-router.js +36 -216
  654. package/src/inference/multi-model-network.js +5 -341
  655. package/src/inference/observation-context.d.ts +22 -0
  656. package/src/inference/observation-context.js +41 -0
  657. package/src/inference/pipelines/audio/index.js +1 -20
  658. package/src/inference/pipelines/audio/mel.js +0 -38
  659. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  660. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  661. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  662. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  663. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  664. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  665. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  666. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  667. package/src/inference/pipelines/factory.js +14 -3
  668. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  669. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  670. package/src/inference/pipelines/text/abort-contract.js +22 -0
  671. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  672. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  673. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  674. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  675. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  676. package/src/inference/pipelines/text/attention/index.js +7 -0
  677. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  678. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  679. package/src/inference/pipelines/text/attention/interpreter.js +984 -0
  680. package/src/inference/pipelines/text/attention/plan.d.ts +37 -0
  681. package/src/inference/pipelines/text/attention/plan.js +345 -0
  682. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  683. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  684. package/src/inference/pipelines/text/attention/projections.js +20 -537
  685. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  686. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  687. package/src/inference/pipelines/text/attention/record.js +1 -1277
  688. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  689. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  690. package/src/inference/pipelines/text/attention/run.js +1 -1385
  691. package/src/inference/pipelines/text/attention/types.js +3 -31
  692. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  693. package/src/inference/pipelines/text/chat-format.js +42 -0
  694. package/src/inference/pipelines/text/config/normalization.js +729 -0
  695. package/src/inference/pipelines/text/config/validation.js +9 -0
  696. package/src/inference/pipelines/text/config.d.ts +12 -1
  697. package/src/inference/pipelines/text/config.js +14 -1228
  698. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  699. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  700. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  701. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  702. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  703. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  704. package/src/inference/pipelines/text/embed.js +178 -241
  705. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  706. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  707. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  708. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  709. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  710. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  711. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  712. package/src/inference/pipelines/text/execution-v1.js +7 -283
  713. package/src/inference/pipelines/text/execution.js +517 -0
  714. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  715. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  716. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  717. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  718. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  719. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  720. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  721. package/src/inference/pipelines/text/generator/decode.js +952 -0
  722. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  723. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  724. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  725. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  726. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  727. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  728. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  729. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  730. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +44 -110
  731. package/src/inference/pipelines/text/generator/text.js +448 -0
  732. package/src/inference/pipelines/text/generator/training.js +752 -0
  733. package/src/inference/pipelines/text/generator-decode-policy.js +5 -2
  734. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  735. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  736. package/src/inference/pipelines/text/generator-steps.js +37 -962
  737. package/src/inference/pipelines/text/generator.js +165 -3539
  738. package/src/inference/pipelines/text/image-transcription.js +304 -0
  739. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  740. package/src/inference/pipelines/text/init-rope.js +37 -189
  741. package/src/inference/pipelines/text/init.d.ts +11 -3
  742. package/src/inference/pipelines/text/init.js +6 -216
  743. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  744. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  745. package/src/inference/pipelines/text/layer-plan-gpu.js +15 -6
  746. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  747. package/src/inference/pipelines/text/layer-plan.js +45 -3
  748. package/src/inference/pipelines/text/layer.js +54 -522
  749. package/src/inference/pipelines/text/lifecycle.js +839 -0
  750. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  751. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  752. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  753. package/src/inference/pipelines/text/linear-attention.js +7 -537
  754. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  755. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  756. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  757. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  758. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  759. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  760. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  761. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  762. package/src/inference/pipelines/text/logits/index.js +62 -102
  763. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  764. package/src/inference/pipelines/text/logits/plan.js +184 -0
  765. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  766. package/src/inference/pipelines/text/logits/readback.js +9 -0
  767. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  768. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  769. package/src/inference/pipelines/text/lora-apply.js +12 -4
  770. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  771. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  772. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  773. package/src/inference/pipelines/text/model-load.js +2 -31
  774. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  775. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  776. package/src/inference/pipelines/text/moe/plan.js +47 -0
  777. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  778. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  779. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  780. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  781. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  782. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  783. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  784. package/src/inference/pipelines/text/probes.js +30 -0
  785. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  786. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  787. package/src/inference/pipelines/text/state.d.ts +14 -0
  788. package/src/inference/pipelines/text/state.js +4 -0
  789. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  790. package/src/inference/pipelines/text/types.d.ts +10 -0
  791. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  792. package/src/inference/pipelines/text.d.ts +2 -2
  793. package/src/inference/pipelines/text.js +48 -1620
  794. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  795. package/src/inference/pipelines/video/index.js +0 -21
  796. package/src/inference/pipelines/vision/attention.js +59 -0
  797. package/src/inference/pipelines/vision/encoder.js +254 -369
  798. package/src/inference/pipelines/vision/gemma4.js +204 -318
  799. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  800. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  801. package/src/inference/pipelines/vision/glmocr.js +557 -0
  802. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  803. package/src/inference/pipelines/vision/index.js +8 -27
  804. package/src/inference/pipelines/vision/ops.js +11 -33
  805. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  806. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  807. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  808. package/src/inference/refactor-receipt.d.ts +49 -0
  809. package/src/inference/refactor-receipt.js +188 -0
  810. package/src/inference/rerank.js +182 -0
  811. package/src/inference/resource-scope.d.ts +33 -0
  812. package/src/inference/resource-scope.js +168 -0
  813. package/src/inference/test-harness.d.ts +3 -0
  814. package/src/inference/test-harness.js +1 -0
  815. package/src/inference/tokenizer.js +1 -1
  816. package/src/inference/tokenizers/behavior-flags.js +6 -3
  817. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  818. package/src/inference/tokenizers/bundled/registry.js +216 -0
  819. package/src/inference/tokenizers/bundled.js +23 -353
  820. package/src/loader/doppler-loader.d.ts +4 -0
  821. package/src/loader/doppler-loader.js +119 -930
  822. package/src/loader/embedding-limit-preflight.js +11 -4
  823. package/src/loader/embedding-loader.js +1 -3
  824. package/src/loader/gpu-tensor-materialization.js +20 -0
  825. package/src/loader/layer-loader.js +15 -119
  826. package/src/loader/layer-tensor-aliases.js +99 -0
  827. package/src/loader/layer-weight-shape.js +29 -0
  828. package/src/loader/load-timing.js +64 -0
  829. package/src/loader/loader-types.d.ts +3 -0
  830. package/src/loader/model-executor.js +819 -0
  831. package/src/loader/model-load-validation.js +15 -0
  832. package/src/loader/quantization-constants.d.ts +11 -29
  833. package/src/loader/quantization-constants.js +1 -2
  834. package/src/loader/shard-resolver.js +0 -14
  835. package/src/loader/tensor-load-error.js +14 -0
  836. package/src/loader/tensor-shard-indices.js +8 -0
  837. package/src/loader/tensors/bf16-loader.js +125 -0
  838. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  839. package/src/loader/tensors/load-executor.js +577 -0
  840. package/src/loader/tensors/source-transform/executor.js +123 -0
  841. package/src/loader/tensors/source-transform/plan.js +107 -0
  842. package/src/loader/tensors/source-transform.js +15 -264
  843. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  844. package/src/loader/tensors/tensor-loader.js +7 -653
  845. package/src/memory/buffer-pool.js +7 -29
  846. package/src/rules/capability-transform-resolver.js +80 -0
  847. package/src/rules/execution-contract.d.ts +73 -0
  848. package/src/rules/execution-contract.js +429 -0
  849. package/src/rules/execution-rules-contract-check.js +1 -1
  850. package/src/rules/generated/rule-bundle.json +4305 -0
  851. package/src/rules/inference/capability-transforms.rules.json +3 -3
  852. package/src/rules/layer-pattern-contract-check.js +1 -1
  853. package/src/rules/rule-registry.js +70 -57
  854. package/src/storage/artifact-storage-context.js +2 -2
  855. package/src/storage/backend-selection.js +43 -0
  856. package/src/storage/distribution-transport.js +5 -0
  857. package/src/storage/download/integrity.d.ts +33 -0
  858. package/src/storage/download/integrity.js +130 -0
  859. package/src/storage/download/resume-inspection.d.ts +22 -0
  860. package/src/storage/download/resume-inspection.js +134 -0
  861. package/src/storage/download/retry.js +57 -0
  862. package/src/storage/download/state.js +131 -0
  863. package/src/storage/download/transport.js +99 -0
  864. package/src/storage/downloader.js +74 -453
  865. package/src/storage/model-cache.d.ts +54 -0
  866. package/src/storage/model-cache.js +590 -0
  867. package/src/storage/shard-manager.d.ts +5 -0
  868. package/src/storage/shard-manager.js +50 -369
  869. package/src/storage/shards/index.js +60 -0
  870. package/src/storage/shards/integrity.js +225 -0
  871. package/src/storage/shards/lifecycle.js +22 -0
  872. package/src/storage/source-artifact-store.js +14 -4
  873. package/src/tooling/boundary-evidence.d.ts +51 -0
  874. package/src/tooling/boundary-evidence.js +408 -0
  875. package/src/tooling/browser-command-runner.js +17 -10
  876. package/src/tooling/capsule-v2.d.ts +11 -0
  877. package/src/tooling/capsule-v2.js +58 -0
  878. package/src/tooling/capsule.js +20 -0
  879. package/src/tooling/command-api-constants.js +1 -1
  880. package/src/tooling/command-api-family-normalizers.js +163 -5
  881. package/src/tooling/command-api.d.ts +42 -3
  882. package/src/tooling/command-api.js +7 -2
  883. package/src/tooling/command-context.d.ts +25 -0
  884. package/src/tooling/command-context.js +53 -0
  885. package/src/tooling/command-envelope.js +1 -1
  886. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  887. package/src/tooling/command-runner.html +49 -1
  888. package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +22 -5
  889. package/src/tooling/command-validation.js +1 -24
  890. package/src/tooling/conversion-run-timing.js +24 -0
  891. package/src/tooling/demo-receipts.d.ts +56 -0
  892. package/src/tooling/demo-receipts.js +113 -0
  893. package/src/tooling/diagnose-runner.js +13 -11
  894. package/src/tooling/distribution-shard-transport.js +11 -0
  895. package/src/tooling/execution-cost-ledger.d.ts +95 -0
  896. package/src/tooling/execution-cost-ledger.js +381 -0
  897. package/src/tooling/hf-registry/catalog-io.js +63 -0
  898. package/src/tooling/litert/package.js +863 -0
  899. package/src/tooling/litert-package-runtime.js +3 -849
  900. package/src/tooling/model-capsule-forge.js +478 -0
  901. package/src/tooling/node-browser/execution.d.ts +40 -0
  902. package/src/tooling/node-browser/execution.js +308 -0
  903. package/src/tooling/node-browser/static-server.js +188 -0
  904. package/src/tooling/node-browser/transport.d.ts +80 -0
  905. package/src/tooling/node-browser/transport.js +622 -0
  906. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  907. package/src/tooling/node-browser-command-runner.js +10 -1112
  908. package/src/tooling/node-capsule-artifact-store.js +44 -0
  909. package/src/tooling/node-command-execution.js +237 -0
  910. package/src/tooling/node-command-runner.js +26 -205
  911. package/src/tooling/node-converter/input.js +521 -0
  912. package/src/tooling/node-converter/output.js +244 -0
  913. package/src/tooling/node-converter-chunk-batches.js +21 -0
  914. package/src/tooling/node-converter.js +7 -720
  915. package/src/tooling/node-source-runtime.js +16 -276
  916. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  917. package/src/tooling/node-webgpu.js +178 -453
  918. package/src/tooling/opfs-cache.d.ts +15 -48
  919. package/src/tooling/opfs-cache.js +8 -510
  920. package/src/tooling/precision-replay-math.d.ts +42 -0
  921. package/src/tooling/precision-replay-math.js +174 -0
  922. package/src/tooling/production-release.d.ts +19 -0
  923. package/src/tooling/production-release.js +647 -0
  924. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  925. package/src/tooling/program-bundle/materialize.js +324 -0
  926. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  927. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  928. package/src/tooling/program-bundle/validation.js +31 -0
  929. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  930. package/src/tooling/program-bundle-parity.d.ts +14 -4
  931. package/src/tooling/program-bundle-parity.js +267 -106
  932. package/src/tooling/program-bundle.d.ts +8 -52
  933. package/src/tooling/program-bundle.js +186 -730
  934. package/src/tooling/provider-v1-contract.d.ts +61 -0
  935. package/src/tooling/provider-v1-contract.js +192 -0
  936. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  937. package/src/tooling/registered-variant-calibration-job.js +210 -0
  938. package/src/tooling/registered-variant-calibration.d.ts +184 -0
  939. package/src/tooling/registered-variant-calibration.js +336 -0
  940. package/src/tooling/runtime-input-composition.js +105 -53
  941. package/src/tooling/runtime-intent.js +41 -0
  942. package/src/tooling/runtime-optimization/candidates.js +664 -0
  943. package/src/tooling/runtime-optimization-campaign.js +166 -0
  944. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  945. package/src/tooling/runtime-optimization-index.js +87 -0
  946. package/src/tooling/runtime-optimization.d.ts +91 -6
  947. package/src/tooling/runtime-optimization.js +290 -414
  948. package/src/tooling/source-intake.d.ts +49 -0
  949. package/src/tooling/source-intake.js +431 -0
  950. package/src/tooling/source-package-profiles.js +1 -1
  951. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  952. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  953. package/src/tooling/source-runtime/node-loader.js +258 -0
  954. package/src/tooling/source-runtime-bundle.js +26 -411
  955. package/src/tooling/source-runtime-converter-config.js +1 -1
  956. package/src/tooling-experimental-exports.js +0 -5
  957. package/src/tooling-exports/evidence.d.ts +45 -0
  958. package/src/tooling-exports/evidence.js +45 -0
  959. package/src/tooling-exports/runtime.d.ts +7 -0
  960. package/src/tooling-exports/runtime.js +7 -0
  961. package/src/tooling-exports/storage.d.ts +5 -0
  962. package/src/tooling-exports/storage.js +4 -0
  963. package/src/tooling-exports.d.ts +14 -0
  964. package/src/tooling-exports.js +8 -3
  965. package/src/tooling-exports.shared.d.ts +46 -0
  966. package/src/tooling-exports.shared.js +37 -0
  967. package/src/training.d.ts +6 -6
  968. package/src/training.js +4 -4
  969. package/src/version.js +1 -1
  970. package/tests/kernels/browser/kernel-suite.js +512 -0
  971. package/tests/kernels/browser/test-page.js +169 -14
  972. package/tests/kernels/harness/tolerance.js +1 -0
  973. package/tests/kernels/reference/index.js +13 -1
  974. package/tests/kernels/reference/vision.js +160 -0
  975. package/src/client/runtime/model-session.js +0 -432
  976. package/src/config/schema/tuner.schema.d.ts +0 -39
  977. package/src/config/schema/tuner.schema.js +0 -13
  978. package/src/converter/quantizer.d.ts +0 -121
  979. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  980. package/src/gpu/kernel-tuner/cache.js +0 -145
  981. package/src/gpu/kernel-tuner/index.js +0 -38
  982. package/src/gpu/kernel-tuner/tuner.js +0 -247
  983. package/src/gpu/kernel-tuner.js +0 -12
  984. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  985. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  986. package/src/gpu/kernels/kernel-tuning.js +0 -158
  987. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  988. package/src/gpu/kernels/utils.d.ts +0 -149
  989. package/src/gpu/profiler.js +0 -382
  990. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
  991. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  992. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  993. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  994. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  995. package/src/rules/converter/execution.rules.json +0 -20
  996. package/src/rules/converter/tensor-roles.rules.json +0 -13
  997. package/src/rules/converter/tokenizer.rules.json +0 -12
  998. package/src/rules/inference/attention.rules.json +0 -69
  999. package/src/rules/inference/config.rules.json +0 -82
  1000. package/src/rules/inference/dtype.rules.json +0 -113
  1001. package/src/rules/inference/execution.rules.json +0 -195
  1002. package/src/rules/inference/ffn.rules.json +0 -123
  1003. package/src/rules/inference/layer-pattern.rules.json +0 -16
  1004. package/src/rules/inference/layer.rules.json +0 -7
  1005. package/src/rules/inference/moe.rules.json +0 -145
  1006. package/src/rules/kernels/attention.rules.json +0 -68
  1007. package/src/rules/kernels/conv2d.rules.json +0 -6
  1008. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  1009. package/src/rules/kernels/dequant.rules.json +0 -58
  1010. package/src/rules/kernels/energy.rules.json +0 -22
  1011. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  1012. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  1013. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  1014. package/src/rules/kernels/gather.rules.json +0 -93
  1015. package/src/rules/kernels/gelu.rules.json +0 -11
  1016. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  1017. package/src/rules/kernels/groupnorm.rules.json +0 -10
  1018. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  1019. package/src/rules/kernels/layernorm.rules.json +0 -6
  1020. package/src/rules/kernels/linear-attention.rules.json +0 -6
  1021. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  1022. package/src/rules/kernels/matmul.rules.json +0 -95
  1023. package/src/rules/kernels/modulate.rules.json +0 -6
  1024. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  1025. package/src/rules/kernels/moe.rules.json +0 -14
  1026. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  1027. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  1028. package/src/rules/kernels/relu.rules.json +0 -6
  1029. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  1030. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  1031. package/src/rules/kernels/residual.rules.json +0 -12
  1032. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  1033. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  1034. package/src/rules/kernels/rope-qk.rules.json +0 -6
  1035. package/src/rules/kernels/rope.rules.json +0 -6
  1036. package/src/rules/kernels/sample.rules.json +0 -6
  1037. package/src/rules/kernels/scale.rules.json +0 -8
  1038. package/src/rules/kernels/silu.rules.json +0 -22
  1039. package/src/rules/kernels/softmax.rules.json +0 -25
  1040. package/src/rules/kernels/split-qg.rules.json +0 -6
  1041. package/src/rules/kernels/split-qkv.rules.json +0 -6
  1042. package/src/rules/kernels/upsample2d.rules.json +0 -6
  1043. package/src/rules/loader/tensor-loader.rules.json +0 -42
  1044. package/src/rules/loader/weights.rules.json +0 -42
  1045. package/src/rules/tooling/command-runtime.rules.json +0 -65
  1046. package/src/tooling/hf-registry-utils.js +0 -395
  1047. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  1048. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  1049. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  1050. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  1051. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  1052. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  1053. /package/src/{utils → formats}/array-buffer.js +0 -0
  1054. /package/src/{utils → formats}/canonical-hash.js +0 -0
  1055. /package/src/{utils → formats}/clone-json.js +0 -0
  1056. /package/src/{utils → formats}/encode-utf8.js +0 -0
  1057. /package/src/{utils → formats}/load-json.js +0 -0
  1058. /package/src/{utils → formats}/plain-object.js +0 -0
  1059. /package/src/{utils → formats}/sha256.js +0 -0
  1060. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  1061. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  1062. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  1063. /package/src/{utils → storage}/runtime-env.js +0 -0
@@ -48,11 +48,13 @@ import {
48
48
  import {
49
49
  buildLayerContext,
50
50
  debugCheckBuffer as debugCheckBufferHelper,
51
+ resolvePerLayerInputsSession,
52
+ } from './generator/session-context.js';
53
+ import {
51
54
  getLogitsConfig,
52
55
  getLogitsWeights,
53
- resolvePerLayerInputsSession,
54
- releaseSharedAttentionState,
55
- } from './generator-helpers.js';
56
+ } from './generator/logits-config.js';
57
+ import { releaseSharedAttentionState } from './generator/attention-lifecycle.js';
56
58
  import {
57
59
  assertTokenIdsInRange,
58
60
  assertTokenIdInRange,
@@ -70,7 +72,11 @@ import { resolveSamplingConfig } from './sampling-config.js';
70
72
  import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
71
73
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
72
74
  import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
73
- import { drainPendingTsirReads } from './tsir-fixture-writer.js';
75
+ import {
76
+ advanceDecodeStepCount,
77
+ createTsirFixtureState,
78
+ drainPendingTsirReads,
79
+ } from './tsir-fixture-writer.js';
74
80
  import {
75
81
  activateFallbackExecutionPlan,
76
82
  hasFallbackExecutionPlan,
@@ -105,15 +111,19 @@ import {
105
111
  // Extracted standalone helpers
106
112
  import {
107
113
  resolvePromptInput,
114
+ } from './generator/prompt-input.js';
115
+ import {
108
116
  releasePerLayerInputBuffer,
117
+ shouldDisablePrefillCommandBatching,
118
+ resolveEffectivePrefillTokenChunkSize,
119
+ } from './generator/prefill-policy.js';
120
+ import {
109
121
  normalizePrefixEmbeddingOverride,
110
122
  resolvePrefillEmbeddingInputIds,
111
123
  resolvePrefillMultimodalBidirectionalSpan,
112
124
  applyPrefixEmbeddingOverride,
113
125
  resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
114
- shouldDisablePrefillCommandBatching,
115
- resolveEffectivePrefillTokenChunkSize,
116
- } from './generator-prefill-helpers.js';
126
+ } from './generator/prefix-embedding.js';
117
127
  import {
118
128
  shouldDisableBatchDecodeAfterShortBatch,
119
129
  resolveHotVocabularyBatchDecodeAvailability,
@@ -129,360 +139,94 @@ import {
129
139
  resolveCurrentKVCacheDtype,
130
140
  cloneRuntimeInferenceWithKVDtype,
131
141
  } from './generator-decode-policy.js';
132
-
133
- const SPECIAL_LIKE_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>\n]{1,32}>)$/i;
134
- const FINITENESS_RESET_WORDS = new Uint32Array(4);
135
- const tokenizerSuppressionCache = new WeakMap();
136
- const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
137
-
138
- function mergeRecordOpLabelCounts(target, source) {
139
- const merged = target && typeof target === 'object' && !Array.isArray(target)
140
- ? target
141
- : {};
142
- if (!source || typeof source !== 'object' || Array.isArray(source)) {
143
- return merged;
144
- }
145
- for (const [label, rawCount] of Object.entries(source)) {
146
- const count = Number(rawCount);
147
- if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
148
- continue;
149
- }
150
- merged[label] = (Number.isFinite(merged[label]) ? merged[label] : 0) + count;
151
- }
152
- return merged;
153
- }
154
-
155
- function recordPrefillRecorderStats(state, recorder) {
156
- if (!recorder || typeof recorder.getStats !== 'function') {
157
- return;
158
- }
159
- const recordStats = recorder.getStats();
160
- state.stats.prefillRecordOps = (state.stats.prefillRecordOps ?? 0) + recordStats.opCount;
161
- state.stats.prefillRecordPasses = (state.stats.prefillRecordPasses ?? 0) + recordStats.computePassCount;
162
- state.stats.prefillRecordOpLabels = mergeRecordOpLabelCounts(
163
- state.stats.prefillRecordOpLabels,
164
- recordStats.opLabelCounts
165
- );
166
- }
167
-
168
- export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
169
- const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
170
- const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
171
- const submit = (runtimeSubmit !== undefined && runtimeSubmit !== null)
172
- ? runtimeSubmit
173
- : manifestSubmit;
174
- if (submit === undefined || submit === null) {
175
- throw new Error('[Pipeline] runtime.inference.session.prefillChunkSubmitMode is required.');
176
- }
177
- if (!PREFILL_CHUNK_SUBMIT_MODES.has(submit)) {
178
- throw new Error(
179
- `[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
180
- );
181
- }
182
- return submit;
183
- }
184
-
185
- function getTokenizerSuppressionCache(tokenizer) {
186
- let cache = tokenizerSuppressionCache.get(tokenizer);
187
- if (!cache) {
188
- cache = new Map();
189
- tokenizerSuppressionCache.set(tokenizer, cache);
190
- }
191
- return cache;
192
- }
193
-
194
- function decodeSingleTokenForSuppression(tokenizer, tokenId) {
195
- if (!tokenizer || typeof tokenizer.decode !== 'function') {
196
- return '';
197
- }
198
- try {
199
- return tokenizer.decode([tokenId], false, false);
200
- } catch {
201
- return '';
202
- }
203
- }
204
-
205
- function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
206
- if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
207
- return [];
208
- }
209
- if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
210
- return [];
211
- }
212
- const cache = getTokenizerSuppressionCache(tokenizer);
213
- const cacheKey = [
214
- vocabSize,
215
- samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
216
- samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
217
- ].join(':');
218
- const cached = cache.get(cacheKey);
219
- if (cached) {
220
- return cached;
221
- }
222
- const tokenIds = [];
223
- for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
224
- const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
225
- && typeof tokenizer.isSpecialToken === 'function'
226
- && tokenizer.isSpecialToken(tokenId);
227
- const specialLike = samplingConfig.suppressSpecialLikeTokens === true
228
- && SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
229
- if (tokenizerSpecial || specialLike) {
230
- tokenIds.push(tokenId);
231
- }
232
- }
233
- cache.set(cacheKey, tokenIds);
234
- return tokenIds;
235
- }
236
-
237
- function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
238
- const vocabSize = state.modelConfig?.vocabSize;
239
- const suppressed = new Set(samplingConfig.suppressTokenIds);
240
- const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
241
- const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
242
- if (Number.isInteger(eosToken)) {
243
- stopTokenIds.add(eosToken);
244
- }
245
- for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
246
- if (!stopTokenIds.has(tokenId)) {
247
- suppressed.add(tokenId);
248
- }
249
- }
250
- return [...suppressed];
251
- }
252
-
253
- function normalizeSelectedLogitTokenIds(value, vocabSize, label) {
254
- if (value == null) {
255
- return null;
256
- }
257
- if (!Array.isArray(value) && !ArrayBuffer.isView(value)) {
258
- throw new Error(`${label} must be an array or typed array.`);
259
- }
260
- const tokenIds = Array.from(value, (entry, index) => {
261
- const tokenId = Number(entry);
262
- if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= vocabSize) {
263
- throw new Error(`${label}[${index}] must be an integer in [0, ${vocabSize}), got "${String(entry)}".`);
264
- }
265
- return tokenId;
266
- });
267
- if (tokenIds.length === 0) {
268
- throw new Error(`${label} must not be empty.`);
269
- }
270
- return tokenIds;
271
- }
272
-
273
- async function traceActivationHealth(label, buffer, dtype, elementCount) {
274
- if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
275
- return;
276
- }
277
- const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
278
- const data = await readBuffer(buffer, elementCount * bytesPerElement);
279
- trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
280
- }
281
-
282
- function ownsBorrowedWeightBuffer(weight) {
283
- return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
284
- }
285
-
286
- function borrowLinearWeight(weight, label) {
287
- if (!weight) {
288
- throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
289
- }
290
- if (isSplitWeightBuffer(weight)) {
291
- throw new Error(
292
- `DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
293
- );
294
- }
295
- return {
296
- value: getPipelineWeightBuffer(weight, label),
297
- owned: ownsBorrowedWeightBuffer(weight),
298
- };
299
- }
300
-
301
- function borrowNormWeight(weight, label) {
302
- if (!weight) {
303
- throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
304
- }
305
- if (isSplitWeightBuffer(weight)) {
306
- throw new Error(
307
- `DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
308
- );
309
- }
310
- return {
311
- value: getPipelineNormWeightBuffer(weight, label),
312
- owned: ownsBorrowedWeightBuffer(weight),
313
- };
314
- }
315
-
316
- function releaseBorrowedWeight(borrowed) {
317
- if (!borrowed?.owned) {
318
- return;
319
- }
320
- const value = borrowed.value;
321
- releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
322
- }
323
-
324
- function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
325
- return logitsState != null
326
- && isWeightBuffer(embeddingWeight)
327
- && getWeightDtype(embeddingWeight) === 'f16'
328
- && getLayout(embeddingWeight) === 'row'
329
- && embeddingTranspose !== true;
330
- }
331
-
332
- function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
333
- const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
334
- if (value == null) {
335
- return undefined;
336
- }
337
- if (!Number.isInteger(value) || value <= 0) {
338
- throw new Error(
339
- 'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
340
- );
341
- }
342
- return value;
343
- }
344
-
345
- function normalizeCanvasTokenIds(canvas, context) {
346
- if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
347
- throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
348
- }
349
- return Array.from(canvas, (value, index) => {
350
- if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
351
- throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
352
- }
353
- return value;
354
- });
355
- }
356
-
357
- function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
358
- if (logits == null) {
359
- return null;
360
- }
361
- const expected = canvasLength * vocabSize;
362
- const values = logits instanceof Float32Array
363
- ? logits
364
- : (Array.isArray(logits) ? Float32Array.from(logits) : null);
365
- if (!values) {
366
- throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
367
- }
368
- if (values.length !== expected) {
369
- throw new Error(
370
- `[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
371
- );
372
- }
373
- return values;
374
- }
375
-
376
- function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
377
- if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
378
- return null;
379
- }
380
- const dtype = logits.logitsDtype;
381
- if (dtype !== 'f32') {
382
- throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
383
- }
384
- if (logits.vocabSize !== vocabSize) {
385
- throw new Error(
386
- `[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
387
- );
388
- }
389
- if (logits.canvasLength !== canvasLength) {
390
- throw new Error(
391
- `[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
392
- );
393
- }
394
- const temperature = logits.temperature ?? 1.0;
395
- if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
396
- throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
397
- }
398
- return {
399
- logitsBuffer: logits.logitsBuffer,
400
- logitsDtype: dtype,
401
- temperature,
402
- releaseOnUse: logits.releaseOnUse !== false,
403
- };
404
- }
405
-
406
- function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
407
- if (state?.kind !== 'soft_embedding') {
408
- return null;
409
- }
410
- if (!isGpuBufferInstance(state.buffer)) {
411
- throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
412
- }
413
- const dtype = state.dtype ?? 'f32';
414
- if (dtype !== 'f32') {
415
- throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
416
- }
417
- if (state.canvasLength !== canvasLength) {
418
- throw new Error(
419
- `[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
420
- `expected ${canvasLength}, got ${state.canvasLength}.`
421
- );
422
- }
423
- if (state.hiddenSize !== hiddenSize) {
424
- throw new Error(
425
- `[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
426
- `expected ${hiddenSize}, got ${state.hiddenSize}.`
427
- );
428
- }
429
- return {
430
- buffer: state.buffer,
431
- dtype,
432
- releaseOnUse: state.releaseOnUse !== false,
433
- scaled: state.scaled === true,
434
- };
435
- }
436
-
437
- let intentBundleModulePromise = null;
438
-
439
- async function getExperimentalIntentBundleModule() {
440
- intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
441
- return intentBundleModulePromise;
442
- }
443
-
142
+ import { FINITENESS_RESET_WORDS, borrowLinearWeight, borrowNormWeight, canUseChunkedSoftEmbeddingLogits, getExperimentalIntentBundleModule, normalizeCanvasTokenIds, normalizeSelectedLogitTokenIds, normalizeSelfConditioningLogits, normalizeSelfConditioningLogitsState, normalizeSelfConditioningSoftEmbeddingState, recordPrefillRecorderStats, releaseBorrowedWeight, resolveDiffusionGemmaSoftEmbeddingChunkRows, resolvePrefillChunkSubmitMode, resolveSuppressedSamplingTokenIds, traceActivationHealth } from './generator/text.js';
143
+ import {
144
+ _createDiffusionGemmaSelfConditioningEmbeddings as _createDiffusionGemmaSelfConditioningEmbeddingsImpl,
145
+ _createDiffusionGemmaSelfConditioningSoftEmbeddingState as _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl,
146
+ computeDiffusionGemmaCanvasLogits as computeDiffusionGemmaCanvasLogitsImpl,
147
+ computeDiffusionGemmaCanvasStep as computeDiffusionGemmaCanvasStepImpl,
148
+ } from './generator/diffusion.js';
149
+ import {
150
+ _replayPrefillDecodeLogits as _replayPrefillDecodeLogitsImpl,
151
+ _shouldUseFinitenessFallback as _shouldUseFinitenessFallbackImpl,
152
+ _recreateKVCacheForExecutionPlan as _recreateKVCacheForExecutionPlanImpl,
153
+ _openFinitenessFallbackWindow as _openFinitenessFallbackWindowImpl,
154
+ _closeFinitenessFallbackWindow as _closeFinitenessFallbackWindowImpl,
155
+ _consumeFinitenessFallbackToken as _consumeFinitenessFallbackTokenImpl,
156
+ _beginFinitenessFallback as _beginFinitenessFallbackImpl,
157
+ _endFinitenessFallback as _endFinitenessFallbackImpl,
158
+ _retryWithFinitenessFallback as _retryWithFinitenessFallbackImpl,
159
+ _retryWithPersistentFinitenessFallback as _retryWithPersistentFinitenessFallbackImpl,
160
+ _retryDecodeStepWithFinitenessWindow as _retryDecodeStepWithFinitenessWindowImpl,
161
+ } from './generator/recovery.js';
162
+ import {
163
+ prefillKVOnly as prefillKVOnlyImpl,
164
+ prefillForLoRATraining as prefillForLoRATrainingImpl,
165
+ prefillWithEmbedding as prefillWithEmbeddingImpl,
166
+ prefillWithLogits as prefillWithLogitsImpl,
167
+ prefillWithTokenLogits as prefillWithTokenLogitsImpl,
168
+ prefillWithTokenLogitsFromKV as prefillWithTokenLogitsFromKVImpl,
169
+ generateWithPrefixKV as generateWithPrefixKVImpl,
170
+ } from './generator/training.js';
171
+ import {
172
+ _generateTokensInternal as _generateTokensInternalImpl,
173
+ generateTokenIds as generateTokenIdsImpl,
174
+ _runDecodeLoop as _runDecodeLoopImpl,
175
+ _decodeStep as _decodeStepImpl,
176
+ decodeStepLogits as decodeStepLogitsImpl,
177
+ advanceWithToken as advanceWithTokenImpl,
178
+ advanceWithTokenAndEmbedding as advanceWithTokenAndEmbeddingImpl,
179
+ _generateNTokensGPU as _generateNTokensGPUImpl,
180
+ } from './generator/decode-runtime.js';
181
+ import {
182
+ _commitPrefillHiddenChunk as _commitPrefillHiddenChunkImpl,
183
+ _prefillInputIdsToLogits as _prefillInputIdsToLogitsImpl,
184
+ _prefillPromptToLogits as _prefillPromptToLogitsImpl,
185
+ _prefillToHidden as _prefillToHiddenImpl,
186
+ _prefill as _prefillImpl,
187
+ } from './generator/prefill-runtime.js';
188
+ export { resolvePrefillChunkSubmitMode } from './generator/text.js';
444
189
 
445
190
  export class PipelineGenerator {
446
191
 
447
- #state;
448
- #finitenessFallbackWindow;
192
+ _state;
193
+ _finitenessFallbackWindow;
449
194
 
450
195
  _assertTokenIdsInRange(tokenIds, context = 'encode') {
451
- assertTokenIdsInRange(this.#state, tokenIds, context);
196
+ assertTokenIdsInRange(this._state, tokenIds, context);
452
197
  }
453
198
 
454
199
  _assertTokenIdInRange(tokenId, context = 'token') {
455
- assertTokenIdInRange(this.#state, tokenId, context);
200
+ assertTokenIdInRange(this._state, tokenId, context);
456
201
  }
457
202
 
458
-
459
203
  constructor(state) {
460
- this.#state = state;
461
- this.#finitenessFallbackWindow = null;
204
+ this._state = state;
205
+ this._finitenessFallbackWindow = null;
462
206
  }
463
207
 
464
208
  _resolveDeferredRoundingWindowTokens() {
465
- const activePlan = resolveActiveExecutionPlan(this.#state);
209
+ const activePlan = resolveActiveExecutionPlan(this._state);
466
210
  return activePlan?.deferredRoundingWindowTokens
467
- ?? resolveDeferredRoundingWindowTokens(this.#state.runtimeConfig?.inference?.compute);
211
+ ?? resolveDeferredRoundingWindowTokens(this._state.runtimeConfig?.inference?.compute);
468
212
  }
469
213
 
470
214
  _getEffectiveActivationDtype() {
471
- return resolveActiveExecutionPlan(this.#state).activationDtype;
215
+ return resolveActiveExecutionPlan(this._state).activationDtype;
472
216
  }
473
217
 
474
218
  _hasFinitenessFallbackWindow() {
475
- return this.#finitenessFallbackWindow !== null;
219
+ return this._finitenessFallbackWindow !== null;
476
220
  }
477
221
 
478
222
  _resetReplayPrefillRuntimeState() {
479
- this.#state.kvCache?.clear?.();
480
- this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
481
- this.#state.currentSeqLen = 0;
223
+ this._state.kvCache?.clear?.();
224
+ this._state.linearAttentionRuntime = resetLinearAttentionRuntime(this._state.linearAttentionRuntime);
225
+ this._state.currentSeqLen = 0;
482
226
  }
483
227
 
484
228
  resetGenerationState() {
485
- if (this.#state.isGenerating) {
229
+ if (this._state.isGenerating) {
486
230
  throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
487
231
  }
488
232
  this._resetReplayPrefillRuntimeState();
@@ -490,120 +234,58 @@ export class PipelineGenerator {
490
234
  }
491
235
 
492
236
  async _replayPrefillDecodeLogits(currentIds, opts) {
493
- // Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
494
- // Without KV cache creation, this bound is not enforced elsewhere.
495
- const kvConfig = this.#state.runtimeConfig?.inference?.session?.kvcache;
496
- const replayMaxSeqLen = kvConfig?.maxSeqLen;
497
- if (Number.isFinite(replayMaxSeqLen) && replayMaxSeqLen > 0 && currentIds.length > replayMaxSeqLen) {
498
- throw new Error(
499
- `[Pipeline] Replay-prefill sequence length ${currentIds.length} exceeds ` +
500
- `runtime.inference.session.kvcache.maxSeqLen (${replayMaxSeqLen}). ` +
501
- 'Increase maxSeqLen in a tier profile or runtime config to allow longer sequences.'
502
- );
503
- }
504
- this.#state.decodeStepCount++;
505
- this._resetReplayPrefillRuntimeState();
506
- const logits = await this._prefill(currentIds, opts);
507
- return {
508
- logits,
509
- logitsBuffer: null,
510
- logitsDtype: null,
511
- rawVocabSize: this.#state.modelConfig.vocabSize,
512
- vocabSize: this.#state.modelConfig.vocabSize,
513
- };
237
+ return _replayPrefillDecodeLogitsImpl.apply(this, arguments);
514
238
  }
515
239
 
516
240
  _shouldUseFinitenessFallback(error, contextLabel) {
517
- if (!shouldRetryWithFinitenessFallback(error)) {
518
- return false;
519
- }
520
- if (!hasFallbackExecutionPlan(this.#state)) {
521
- throw createUnhandledFinitenessPolicyError(this.#state, contextLabel, error);
522
- }
523
- return true;
241
+ return _shouldUseFinitenessFallbackImpl.apply(this, arguments);
524
242
  }
525
243
 
526
244
  _recreateKVCacheForExecutionPlan(plan, reasonLabel) {
527
- const kvDtype = resolveTargetPlanKVDtype(plan, `${reasonLabel}: target plan`);
528
- const runtimeInference = cloneRuntimeInferenceWithKVDtype(this.#state, kvDtype);
529
- this.#state.kvCache?.destroy?.();
530
- this.#state.kvCache = createKVCache(
531
- this.#state.modelConfig,
532
- this.#state.useGPU,
533
- this.#state.debug,
534
- runtimeInference
535
- );
536
- this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
537
- this.#state.currentSeqLen = 0;
538
- return kvDtype;
245
+ return _recreateKVCacheForExecutionPlanImpl.apply(this, arguments);
539
246
  }
540
247
 
541
248
  _openFinitenessFallbackWindow(opts, reasonLabel, tokenCount, rollbackSeqLen = undefined) {
542
- const normalizedCount = Number.isFinite(tokenCount)
543
- ? Math.max(1, Math.floor(tokenCount))
544
- : 1;
545
- if (this.#finitenessFallbackWindow) {
546
- this.#finitenessFallbackWindow.remainingTokens = Math.max(
547
- this.#finitenessFallbackWindow.remainingTokens,
548
- normalizedCount
549
- );
550
- return;
551
- }
552
- const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
553
- this.#finitenessFallbackWindow = {
554
- original,
555
- remainingTokens: normalizedCount,
556
- };
249
+ return _openFinitenessFallbackWindowImpl.apply(this, arguments);
557
250
  }
558
251
 
559
252
  _closeFinitenessFallbackWindow(opts) {
560
- if (!this.#finitenessFallbackWindow) {
561
- return;
562
- }
563
- const original = this.#finitenessFallbackWindow.original;
564
- this.#finitenessFallbackWindow = null;
565
- this._endFinitenessFallback(opts, original);
253
+ return _closeFinitenessFallbackWindowImpl.apply(this, arguments);
566
254
  }
567
255
 
568
256
  _consumeFinitenessFallbackToken(opts) {
569
- if (!this.#finitenessFallbackWindow) {
570
- return;
571
- }
572
- this.#finitenessFallbackWindow.remainingTokens -= 1;
573
- if (this.#finitenessFallbackWindow.remainingTokens <= 0) {
574
- this._closeFinitenessFallbackWindow(opts);
575
- }
257
+ return _consumeFinitenessFallbackTokenImpl.apply(this, arguments);
576
258
  }
577
259
 
578
260
  _resolveStepOptions(options = {}) {
579
- return resolveStepOptions(this.#state, options);
261
+ return resolveStepOptions(this._state, options);
580
262
  }
581
263
 
582
264
  _resetDecodeRuntimeState() {
583
- this.#state.stats.prefillProfileSteps = [];
584
- this.#state.stats.decodeMode = null;
585
- this.#state.stats.batchGuardReason = null;
586
- this.#state.stats.decodeProfileSteps = [];
587
- this.#state.stats.ttftMs = 0;
588
- this.#state.stats.decodeTimeMs = 0;
589
- this.#state.stats.prefillRecordMs = 0;
590
- this.#state.stats.prefillRecordOps = 0;
591
- this.#state.stats.prefillRecordPasses = 0;
592
- this.#state.stats.prefillRecordOpLabels = {};
593
- this.#state.stats.prefillSubmitWaitMs = 0;
594
- this.#state.stats.decodeRecordMs = 0;
595
- this.#state.stats.decodeRecordOps = 0;
596
- this.#state.stats.decodeRecordPasses = 0;
597
- this.#state.stats.decodeRecordOpLabels = {};
598
- this.#state.stats.decodeSubmitWaitMs = 0;
599
- this.#state.stats.decodeReadbackWaitMs = 0;
600
- this.#state.stats.decodeReadbackMapWaitMs = 0;
601
- this.#state.stats.decodeReadbackCleanupMs = 0;
602
- this.#state.stats.decodeReadbackCopyMs = 0;
603
- this.#state.decodeStepCount = 0;
604
- this.#state.disableRecordedLogits = false;
605
- this.#state.disableFusedDecode = false;
606
- this.#state.batchingStats = {
265
+ this._state.stats.prefillProfileSteps = [];
266
+ this._state.stats.decodeMode = null;
267
+ this._state.stats.batchGuardReason = null;
268
+ this._state.stats.decodeProfileSteps = [];
269
+ this._state.stats.ttftMs = 0;
270
+ this._state.stats.decodeTimeMs = 0;
271
+ this._state.stats.prefillRecordMs = 0;
272
+ this._state.stats.prefillRecordOps = 0;
273
+ this._state.stats.prefillRecordPasses = 0;
274
+ this._state.stats.prefillRecordOpLabels = {};
275
+ this._state.stats.prefillSubmitWaitMs = 0;
276
+ this._state.stats.decodeRecordMs = 0;
277
+ this._state.stats.decodeRecordOps = 0;
278
+ this._state.stats.decodeRecordPasses = 0;
279
+ this._state.stats.decodeRecordOpLabels = {};
280
+ this._state.stats.decodeSubmitWaitMs = 0;
281
+ this._state.stats.decodeReadbackWaitMs = 0;
282
+ this._state.stats.decodeReadbackMapWaitMs = 0;
283
+ this._state.stats.decodeReadbackCleanupMs = 0;
284
+ this._state.stats.decodeReadbackCopyMs = 0;
285
+ this._state.decodeStepCount = 0;
286
+ this._state.disableRecordedLogits = false;
287
+ this._state.disableFusedDecode = false;
288
+ this._state.batchingStats = {
607
289
  batchedForwardCalls: 0,
608
290
  unbatchedForwardCalls: 0,
609
291
  totalBatchedTimeMs: 0,
@@ -616,23 +298,23 @@ export class PipelineGenerator {
616
298
  maxBatchTokenCap: null,
617
299
  batchClampCount: 0,
618
300
  };
619
- resetActiveExecutionPlan(this.#state);
620
- this.#state.decodeRing?.reset();
301
+ resetActiveExecutionPlan(this._state);
302
+ this._state.decodeRing?.reset();
621
303
  }
622
304
 
623
305
  _getDecodeHelpers(debugCheckBuffer) {
624
306
  return {
625
307
  buildLayerContext: (recorder, isDecodeMode, debugLayers, executionPlan) =>
626
- buildLayerContext(this.#state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
627
- getLogitsWeights: () => getLogitsWeights(this.#state),
628
- getLogitsConfig: () => getLogitsConfig(this.#state),
308
+ buildLayerContext(this._state, recorder, isDecodeMode, debugLayers, debugCheckBuffer, executionPlan),
309
+ getLogitsWeights: () => getLogitsWeights(this._state),
310
+ getLogitsConfig: () => getLogitsConfig(this._state),
629
311
  releaseSharedAttentionState,
630
312
  debugCheckBuffer,
631
313
  };
632
314
  }
633
315
 
634
316
  async _getFinalNormWeights() {
635
- return getFinalNormWeights(this.#state);
317
+ return getFinalNormWeights(this._state);
636
318
  }
637
319
 
638
320
  _extractEmbeddingFromHidden(hiddenStates, numTokens, hiddenSize, embeddingMode, finalNormWeights, config) {
@@ -643,9 +325,9 @@ export class PipelineGenerator {
643
325
  embeddingMode,
644
326
  finalNormWeights,
645
327
  config,
646
- this.#state.embeddingPostprocessor,
328
+ this._state.embeddingPostprocessor,
647
329
  null,
648
- this.#state.weights.get('final_norm_bias') ?? null
330
+ this._state.weights.get('final_norm_bias') ?? null
649
331
  );
650
332
  }
651
333
 
@@ -656,13 +338,13 @@ export class PipelineGenerator {
656
338
  hiddenSize,
657
339
  finalNormWeights,
658
340
  config,
659
- this.#state.weights.get('final_norm_bias') ?? null
341
+ this._state.weights.get('final_norm_bias') ?? null
660
342
  );
661
343
  }
662
344
 
663
345
  _resolvePromptTokenIds(prompt, useChatTemplate, contextLabel) {
664
- const processedPrompt = resolvePromptInput(this.#state, prompt, useChatTemplate, contextLabel);
665
- const inputIds = this.#state.tokenizer.encode(processedPrompt);
346
+ const processedPrompt = resolvePromptInput(this._state, prompt, useChatTemplate, contextLabel);
347
+ const inputIds = this._state.tokenizer.encode(processedPrompt);
666
348
  this._assertTokenIdsInRange(inputIds, `${contextLabel}.encode`);
667
349
  return inputIds;
668
350
  }
@@ -686,14 +368,14 @@ export class PipelineGenerator {
686
368
  try {
687
369
  opts.logitMaskFn(sampledLogits, {
688
370
  generatedIds,
689
- tokenizer: this.#state.tokenizer ?? null,
690
- vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
371
+ tokenizer: this._state.tokenizer ?? null,
372
+ vocabSize: this._state.modelConfig?.vocabSize ?? sampledLogits.length,
691
373
  });
692
374
  } catch (maskError) {
693
375
  log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
694
376
  }
695
377
  }
696
- const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
378
+ const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
697
379
  const tokenId = sample(sampledLogits, {
698
380
  temperature: opts.temperature,
699
381
  topP: opts.topP,
@@ -712,7 +394,7 @@ export class PipelineGenerator {
712
394
  }
713
395
 
714
396
  _resolvePrefillTokenChunkSize(inputIds) {
715
- const chunkSize = resolveEffectivePrefillTokenChunkSize(this.#state);
397
+ const chunkSize = resolveEffectivePrefillTokenChunkSize(this._state);
716
398
  if (chunkSize === undefined) {
717
399
  throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
718
400
  }
@@ -726,105 +408,26 @@ export class PipelineGenerator {
726
408
  }
727
409
 
728
410
  async _commitPrefillHiddenChunk(prefillResult) {
729
- const {
730
- numTokens,
731
- startPos,
732
- currentRecorder,
733
- recordProfile,
734
- currentHiddenBuffer,
735
- } = prefillResult;
736
-
737
- try {
738
- if (currentRecorder) {
739
- recordPrefillRecorderStats(this.#state, currentRecorder);
740
- await currentRecorder.submitAndWait();
741
- await recordProfile(currentRecorder);
742
- } else {
743
- const device = getDevice();
744
- if (device) {
745
- await device.queue.onSubmittedWorkDone();
746
- }
747
- }
748
- this.#state.currentSeqLen = startPos + numTokens;
749
- } finally {
750
- releaseBuffer(currentHiddenBuffer);
751
- }
411
+ return _commitPrefillHiddenChunkImpl.apply(this, arguments);
752
412
  }
753
413
 
754
414
  async _prefillInputIdsToLogits(inputIds, opts) {
755
- const chunkSize = this._resolvePrefillTokenChunkSize(inputIds);
756
- if (chunkSize === null) {
757
- return this._prefill(inputIds, opts);
758
- }
759
-
760
- for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
761
- const end = Math.min(offset + chunkSize, inputIds.length);
762
- const chunk = inputIds.slice(offset, end);
763
- const isFinalChunk = end === inputIds.length;
764
- if (isFinalChunk) {
765
- return this._prefill(chunk, opts);
766
- }
767
- const prefillResult = await this._prefillToHidden(chunk, opts);
768
- await this._commitPrefillHiddenChunk(prefillResult);
769
- }
770
-
771
- return this._prefill(inputIds, opts);
415
+ return _prefillInputIdsToLogitsImpl.apply(this, arguments);
772
416
  }
773
417
 
774
418
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
775
- const prefillStartSeqLen = this.#state.currentSeqLen;
776
- const inputStart = performance.now();
777
- const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
778
- const inputMs = performance.now() - inputStart;
779
- if (opts.debug) {
780
- log.debug('Pipeline', `${contextLabel}: ${inputIds.length} tokens`);
781
- }
782
-
783
- let logits;
784
- const prefillStart = performance.now();
785
- const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
786
- try {
787
- logits = await runPrefill();
788
- } catch (error) {
789
- if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
790
- throw error;
791
- }
792
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during ${contextLabel}. Retrying with F32 precision.`);
793
- logits = await this._retryWithPersistentFinitenessFallback(
794
- opts,
795
- contextLabel,
796
- opts.maxTokens ?? 1,
797
- runPrefill,
798
- prefillStartSeqLen
799
- );
800
- }
801
-
802
- return {
803
- inputIds,
804
- logits,
805
- phase: {
806
- inputMs,
807
- prefillMs: performance.now() - prefillStart,
808
- tokens: inputIds.length,
809
- prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
810
- prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
811
- prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
812
- prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
813
- prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
814
- gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
815
- },
816
- };
419
+ return _prefillPromptToLogitsImpl.apply(this, arguments);
817
420
  }
818
421
 
819
422
  async _decodeStepToLogits(currentIds, opts) {
820
- if (usesReplayPrefillDecode(this.#state)) {
423
+ if (usesReplayPrefillDecode(this._state)) {
821
424
  return this._replayPrefillDecodeLogits(currentIds, opts);
822
425
  }
823
- const debugCheckBuffer = this.#state.debug
426
+ const debugCheckBuffer = this._state.debug
824
427
  ? (buffer, label, numTokens, expectedDim) =>
825
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
428
+ debugCheckBufferHelper(this._state, buffer, label, numTokens, expectedDim)
826
429
  : undefined;
827
- return decodeStepLogits(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
430
+ return decodeStepLogits(this._state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
828
431
  }
829
432
 
830
433
  async _decodeNextTokenViaLogits(currentIds, opts) {
@@ -836,7 +439,7 @@ export class PipelineGenerator {
836
439
  if (!Array.isArray(stopSequences) || stopSequences.length === 0) {
837
440
  return false;
838
441
  }
839
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
442
+ const fullText = this._state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
840
443
  return stopSequences.some((sequence) => fullText.endsWith(sequence));
841
444
  }
842
445
 
@@ -848,429 +451,32 @@ export class PipelineGenerator {
848
451
  }
849
452
 
850
453
  _recordStopReason(reason, tokenId = null) {
851
- this.#state.stats.stopReason = reason;
852
- this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
454
+ this._state.stats.stopReason = reason;
455
+ this._state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
853
456
  }
854
457
 
855
458
  async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
856
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
857
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
858
-
859
- validateCallTimeOptions(options);
860
-
861
- this.#state.isGenerating = true;
862
- this._resetDecodeRuntimeState();
863
- this.#state.stats.gpuTimePrefillMs = undefined;
864
- this.#state.stats.gpuTimeDecodeMs = undefined;
865
- this.#state.stats.decodeRecordMs = 0;
866
- this.#state.stats.decodeRecordOps = 0;
867
- this.#state.stats.decodeRecordPasses = 0;
868
- this.#state.stats.decodeRecordOpLabels = {};
869
- this.#state.stats.decodeSubmitWaitMs = 0;
870
- this.#state.stats.decodeReadbackWaitMs = 0;
871
- this.#state.stats.decodeReadbackMapWaitMs = 0;
872
- this.#state.stats.decodeReadbackCleanupMs = 0;
873
- this.#state.stats.decodeReadbackCopyMs = 0;
874
- this.#state.stats.prefillRecordMs = 0;
875
- this.#state.stats.prefillRecordOps = 0;
876
- this.#state.stats.prefillRecordPasses = 0;
877
- this.#state.stats.prefillRecordOpLabels = {};
878
- this.#state.stats.prefillSubmitWaitMs = 0;
879
- this.#state.stats.singleTokenSubmitWaitMs = 0;
880
- this.#state.stats.singleTokenReadbackWaitMs = 0;
881
- this.#state.stats.singleTokenReadbackMapWaitMs = 0;
882
- this.#state.stats.singleTokenReadbackCleanupMs = 0;
883
- this.#state.stats.singleTokenReadbackCopyMs = 0;
884
- this.#state.stats.singleTokenOrchestrationMs = 0;
885
- this.#state.stats.pleHotVocabularyHits = 0;
886
- this.#state.stats.pleHotVocabularyMisses = 0;
887
- this.#state.stats.plePreparedTokenCacheHits = 0;
888
- this.#state.stats.plePreparedTokenCacheMisses = 0;
889
- this.#state.stats.plePreparedTokenCacheEntries = 0;
890
- this.#state.stats.plePreparedTokenCacheBytes = 0;
891
- this.#state.stats.decodeMode = null;
892
- this.#state.stats.batchGuardReason = null;
893
- this.#state.stats.stopReason = null;
894
- this.#state.stats.stopTokenId = null;
895
- this.#state.stats.ttftMs = 0;
896
- const startTime = performance.now();
897
-
898
- const opts = resolveGenerateOptions(this.#state, options);
899
- opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
900
- opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
901
- // Validate and normalize sampling parameters through single source of truth
902
- const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
903
- opts.temperature = samplingConfig.temperature;
904
- opts.topP = samplingConfig.topP;
905
- opts.topK = samplingConfig.topK;
906
- opts.repetitionPenalty = samplingConfig.repetitionPenalty;
907
- opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
908
- const diagnosticsEnabled = options?.diagnostics?.enabled === true
909
- || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
910
- const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
911
- if (diagnosticsEnabled || tsirFixtureCfg) {
912
- const captureConfig = {
913
- ...createDefaultCaptureConfig(),
914
- enabled: true,
915
- defaultLevel: CAPTURE_LEVELS.SLICE,
916
- ...(options?.diagnostics?.captureConfig ?? {}),
917
- };
918
- validateCaptureConfig(captureConfig);
919
- this.#state.operatorDiagnostics = {
920
- enabled: diagnosticsEnabled === true,
921
- captureConfig,
922
- emitter: diagnosticsEnabled ? new OperatorEventEmitter({
923
- modelHash: this.#state.manifest?.modelId ?? null,
924
- runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
925
- executionPlanHash: opts.executionPlan?.id ?? null,
926
- }) : null,
927
- tsirFixture: tsirFixtureCfg ? {
928
- dir: tsirFixtureCfg.dir,
929
- layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
930
- ? tsirFixtureCfg.layerFilter
931
- : null,
932
- records: [],
933
- } : null,
934
- };
935
- }
936
- const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
937
- this.#state.stats.executionPlan = {
938
- primary: summarizeExecutionPlan(this.#state.executionPlanState?.primaryPlan ?? null),
939
- fallback: summarizeExecutionPlan(this.#state.executionPlanState?.fallbackPlan ?? null),
940
- activePlanIdAtStart: activePlan?.id ?? null,
941
- finalActivePlanId: activePlan?.id ?? null,
942
- transitions: [],
943
- };
944
- this.#state.stats.kernelPathId = activePlan?.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
945
- this.#state.stats.kernelPathSource = activePlan?.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
946
-
947
- if (opts.debug) {
948
- log.debug('Pipeline', `ChatTemplate: options=${options.useChatTemplate}, final=${opts.useChatTemplate}`);
949
- }
950
-
951
- const emitToken = async function* (generator, tokenId, textDecoder) {
952
- if (mode === 'token') {
953
- yield tokenId;
954
- if (options.onToken) options.onToken(tokenId, '');
955
- return;
956
- }
957
- const tokenText = textDecoder(tokenId);
958
- yield tokenText;
959
- if (options.onToken) options.onToken(tokenId, tokenText);
960
- };
961
-
962
- try {
963
- const prefillStartSeqLen = this.#state.currentSeqLen;
964
- const prefillStart = performance.now();
965
- const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generate');
966
- let prefillLogits = initialPrefillLogits;
967
- this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
968
- this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
969
- const generatedIds = [...inputIds];
970
- this.#state.stats.prefillTokens = inputIds.length;
971
-
972
- if (opts.debug) {
973
- log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
974
- }
975
-
976
- const intentBundleConfig = this.#state.runtimeConfig.shared.intentBundle;
977
- const intentBundle = intentBundleConfig?.bundle;
978
- const expectedTopK = intentBundle?.payload?.expectedTopK
979
- ?? intentBundle?.payload?.expected_top_k;
980
- const maxDriftThreshold = intentBundle?.constraints?.maxDriftThreshold
981
- ?? intentBundle?.constraints?.max_drift_threshold;
982
-
983
- if (intentBundleConfig?.enabled && Array.isArray(expectedTopK) && expectedTopK.length > 0) {
984
- const { enforceLogitDrift } = await getExperimentalIntentBundleModule();
985
- const actualTopK = getTopK(
986
- prefillLogits,
987
- expectedTopK.length,
988
- (tokens) => resolveTokenText(this.#state.tokenizer, tokens),
989
- ).map((token) => token.token);
990
- const driftResult = enforceLogitDrift(expectedTopK, actualTopK, maxDriftThreshold);
991
- if (!driftResult.ok) {
992
- throw new Error(`Intent bundle drift check failed: ${driftResult.reason}`);
993
- }
994
- }
995
-
996
- if (opts.debug) {
997
- const topAfterPenalty = getTopK(
998
- Float32Array.from(prefillLogits),
999
- 5,
1000
- (tokens) => resolveTokenText(this.#state.tokenizer, tokens)
1001
- );
1002
- log.debug('Pipeline', `After rep penalty top-5: ${topAfterPenalty.map(t => `"${t.text}"(${(t.prob * 100).toFixed(1)}%)`).join(', ')}`);
1003
- }
1004
-
1005
- let firstToken;
1006
- try {
1007
- firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
1008
- } catch (error) {
1009
- if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
1010
- throw error;
1011
- }
1012
- log.warn('Pipeline', 'FinitenessGuard caught non-finite prefill logits at sampling. Retrying with F32 precision.');
1013
- prefillLogits = await this._retryWithPersistentFinitenessFallback(
1014
- opts,
1015
- 'prefill-sample',
1016
- opts.maxTokens,
1017
- () => this._prefill(inputIds, opts),
1018
- prefillStartSeqLen
1019
- );
1020
- firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
1021
- }
1022
-
1023
- if (opts.debug) {
1024
- const firstTokenText = resolveTokenText(this.#state.tokenizer, [firstToken], `[${firstToken}]`, (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false));
1025
- log.debug('Pipeline', `First token sampled: id=${firstToken} text="${firstTokenText}"`);
1026
- }
1027
-
1028
- const stopTokenIds = this.#state.modelConfig.stopTokenIds;
1029
- const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
1030
- const stopSequenceStart = inputIds.length;
1031
- generatedIds.push(firstToken);
1032
- this.#state.stats.ttftMs = performance.now() - startTime;
1033
-
1034
- const decodeToken = (tokenId) => resolveTokenText(
1035
- this.#state.tokenizer,
1036
- [tokenId],
1037
- `[${tokenId}]`,
1038
- (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
1039
- (tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
1040
- );
1041
- const decodeRuntime = {
1042
- stopTokenIds,
1043
- eosToken,
1044
- stopSequenceStart,
1045
- decodeToken,
1046
- logBatchPath: opts.debug,
1047
- emitMode: mode,
1048
- };
1049
-
1050
- yield* emitToken(this, firstToken, decodeToken);
1051
-
1052
- if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1053
- this._recordStopReason('stop-token-or-sequence', firstToken);
1054
- this.#state.stats.decodeTimeMs = 0;
1055
- this.#state.stats.tokensGenerated = 1;
1056
- this.#state.stats.decodeTokens = 1;
1057
- } else {
1058
- yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
1059
- }
1060
- const tokensGenerated = this.#state.stats.decodeTokens ?? 1;
1061
- this.#state.stats.totalTimeMs = performance.now() - startTime;
1062
-
1063
- if (opts.debug) {
1064
- log.debug('Pipeline', `Generated ${tokensGenerated} tokens in ${this.#state.stats.totalTimeMs.toFixed(0)}ms`);
1065
- }
1066
-
1067
- const ttft = this.#state.stats.ttftMs ?? this.#state.stats.prefillTimeMs;
1068
- const decodeTokens = Math.max(0, tokensGenerated - 1);
1069
- const decodeSpeed = decodeTokens > 0 ? (decodeTokens / this.#state.stats.decodeTimeMs * 1000) : 0;
1070
- const loadMs = this.#state.stats.modelLoadMs;
1071
- const loadLabel = Number.isFinite(loadMs) ? `Load: ${loadMs.toFixed(0)}ms | ` : '';
1072
- if (opts.benchmark) {
1073
- log.info('Benchmark', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
1074
- } else {
1075
- log.info('Perf', `${loadLabel}TTFT: ${ttft.toFixed(0)}ms | Prefill: ${this.#state.stats.prefillTimeMs.toFixed(0)}ms | Decode: ${this.#state.stats.decodeTimeMs.toFixed(0)}ms (${decodeTokens} tokens @ ${decodeSpeed.toFixed(1)} tok/s)`);
1076
- }
1077
- trace.perf('Decode summary', {
1078
- ttftMs: ttft,
1079
- prefillMs: this.#state.stats.prefillTimeMs,
1080
- decodeMs: this.#state.stats.decodeTimeMs,
1081
- decodeTokens,
1082
- decodeSpeed,
1083
- totalMs: this.#state.stats.totalTimeMs,
1084
- });
1085
- } finally {
1086
- this._closeFinitenessFallbackWindow(opts);
1087
- resetActiveExecutionPlan(this.#state);
1088
- this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
1089
- ? {
1090
- enabled: true,
1091
- timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
1092
- recordCount: this.#state.operatorDiagnostics.emitter.length,
1093
- }
1094
- : null;
1095
- this.#state.operatorDiagnostics = null;
1096
- this.#state.isGenerating = false;
1097
- }
459
+ return yield* _generateTokensInternalImpl.apply(this, arguments);
1098
460
  }
1099
461
 
1100
462
  _beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen = undefined) {
1101
- const originalPlan = resolveActiveExecutionPlan(this.#state);
1102
- const currentKvDtype = resolveCurrentKVCacheDtype(
1103
- this.#state,
1104
- originalPlan,
1105
- `${reasonLabel}: current plan`
1106
- );
1107
- const original = {
1108
- activePlanId: this.#state.executionPlanState?.activePlanId ?? 'primary',
1109
- seed: opts.seed,
1110
- restoreKVCachePlan: null,
1111
- };
1112
-
1113
- const fallbackPlan = activateFallbackExecutionPlan(this.#state);
1114
- if (!fallbackPlan) {
1115
- throw new Error(
1116
- '[Pipeline] Explicit alternate-plan finiteness recovery is unavailable for this model/runtime configuration.'
1117
- );
1118
- }
1119
- log.warn(
1120
- 'Pipeline',
1121
- `FinitenessGuard fallback (${reasonLabel}): ` +
1122
- `${originalPlan.kernelPathId ?? 'none'} -> ${fallbackPlan.kernelPathId ?? 'none'}`
1123
- );
1124
- const fallbackKvDtype = resolveTargetPlanKVDtype(
1125
- fallbackPlan,
1126
- `${reasonLabel}: fallback plan`
1127
- );
1128
-
1129
- if (Number.isInteger(rollbackSeqLen) && rollbackSeqLen < 0) {
1130
- setActiveExecutionPlan(this.#state, original.activePlanId);
1131
- throw new Error(
1132
- `[Pipeline] ${reasonLabel}: rollbackSeqLen must be a non-negative integer when provided.`
1133
- );
1134
- }
1135
-
1136
- if (fallbackKvDtype !== currentKvDtype) {
1137
- if (rollbackSeqLen !== 0) {
1138
- setActiveExecutionPlan(this.#state, original.activePlanId);
1139
- throw new Error(
1140
- `[Pipeline] ${reasonLabel}: finiteness fallback requires rebuilding the KV cache ` +
1141
- `${currentKvDtype} -> ${fallbackKvDtype}, which is only supported from a fresh prefill (rollbackSeqLen=0).`
1142
- );
1143
- }
1144
- try {
1145
- this._recreateKVCacheForExecutionPlan(fallbackPlan, reasonLabel);
1146
- original.restoreKVCachePlan = originalPlan;
1147
- } catch (error) {
1148
- setActiveExecutionPlan(this.#state, original.activePlanId);
1149
- try {
1150
- this._recreateKVCacheForExecutionPlan(originalPlan, `${reasonLabel}: restore primary`);
1151
- } catch (restoreError) {
1152
- log.warn(
1153
- 'Pipeline',
1154
- `Failed to restore primary KV cache after fallback activation error: ${restoreError}`
1155
- );
1156
- }
1157
- throw error;
1158
- }
1159
- } else if (Number.isInteger(rollbackSeqLen)) {
1160
- this.#state.kvCache?.truncate(rollbackSeqLen);
1161
- this.#state.currentSeqLen = rollbackSeqLen;
1162
- if (rollbackSeqLen === 0) {
1163
- this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
1164
- }
1165
- } else {
1166
- this.#state.kvCache?.truncate(this.#state.currentSeqLen);
1167
- }
1168
-
1169
- this.#state.decodeBuffers?.ensureBuffers({
1170
- hiddenSize: this.#state.modelConfig.hiddenSize,
1171
- intermediateSize: this.#state.modelConfig.maxIntermediateSize,
1172
- activationDtype: fallbackPlan.activationDtype,
1173
- enablePingPong: true,
1174
- });
1175
-
1176
- if (opts.seed == null) {
1177
- const fallbackSeedBase = (this.#state.decodeStepCount + this.#state.currentSeqLen + 1) >>> 0;
1178
- opts.seed = (fallbackSeedBase * 2654435761) >>> 0;
1179
- }
1180
- opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
1181
- if (this.#state.stats.executionPlan) {
1182
- this.#state.stats.executionPlan.finalActivePlanId = fallbackPlan.id;
1183
- this.#state.stats.executionPlan.transitions.push({
1184
- kind: 'activate-finiteness-fallback',
1185
- reason: reasonLabel ?? null,
1186
- decodeStep: this.#state.decodeStepCount,
1187
- seqLen: this.#state.currentSeqLen,
1188
- fromPlanId: originalPlan.id,
1189
- toPlanId: fallbackPlan.id,
1190
- fromKernelPathId: originalPlan.kernelPathId ?? null,
1191
- toKernelPathId: fallbackPlan.kernelPathId ?? null,
1192
- });
1193
- }
1194
- this.#state.stats.kernelPathId = fallbackPlan.kernelPathId ?? null;
1195
- this.#state.stats.kernelPathSource = fallbackPlan.kernelPathSource ?? 'none';
1196
-
1197
- return original;
463
+ return _beginFinitenessFallbackImpl.apply(this, arguments);
1198
464
  }
1199
465
 
1200
466
  _endFinitenessFallback(opts, original) {
1201
- opts.seed = original.seed;
1202
- setActiveExecutionPlan(this.#state, original.activePlanId);
1203
- opts.executionPlan = rebaseExecutionSessionPlan(this.#state, opts.executionPlan);
1204
- const restoredPlan = resolveActiveExecutionPlan(this.#state);
1205
- if (original.restoreKVCachePlan) {
1206
- this._recreateKVCacheForExecutionPlan(restoredPlan, 'restore-primary-plan');
1207
- }
1208
- if (this.#state.stats.executionPlan) {
1209
- this.#state.stats.executionPlan.finalActivePlanId = restoredPlan.id;
1210
- this.#state.stats.executionPlan.transitions.push({
1211
- kind: 'restore-primary-plan',
1212
- reason: null,
1213
- decodeStep: this.#state.decodeStepCount,
1214
- seqLen: this.#state.currentSeqLen,
1215
- fromPlanId: this.#state.executionPlanState?.fallbackPlan?.id ?? null,
1216
- toPlanId: restoredPlan.id,
1217
- fromKernelPathId: this.#state.executionPlanState?.fallbackPlan?.kernelPathId ?? null,
1218
- toKernelPathId: restoredPlan.kernelPathId ?? null,
1219
- });
1220
- }
1221
- this.#state.stats.kernelPathId = restoredPlan.kernelPathId ?? this.#state.resolvedKernelPath?.id ?? null;
1222
- this.#state.stats.kernelPathSource = restoredPlan.kernelPathSource ?? this.#state.kernelPathSource ?? 'none';
1223
- const nextActivationDtype = this._getEffectiveActivationDtype();
1224
- this.#state.decodeBuffers?.ensureBuffers({
1225
- hiddenSize: this.#state.modelConfig.hiddenSize,
1226
- intermediateSize: this.#state.modelConfig.maxIntermediateSize,
1227
- activationDtype: nextActivationDtype,
1228
- enablePingPong: true,
1229
- });
467
+ return _endFinitenessFallbackImpl.apply(this, arguments);
1230
468
  }
1231
469
 
1232
470
  async _retryWithFinitenessFallback(opts, reasonLabel, retryFn, rollbackSeqLen = undefined) {
1233
- if (this._hasFinitenessFallbackWindow()) {
1234
- return retryFn();
1235
- }
1236
- const original = this._beginFinitenessFallback(opts, reasonLabel, rollbackSeqLen);
1237
- try {
1238
- return await retryFn();
1239
- } finally {
1240
- this._endFinitenessFallback(opts, original);
1241
- }
471
+ return _retryWithFinitenessFallbackImpl.apply(this, arguments);
1242
472
  }
1243
473
 
1244
474
  async _retryWithPersistentFinitenessFallback(opts, reasonLabel, tokenBudget, retryFn, rollbackSeqLen = undefined) {
1245
- if (this._hasFinitenessFallbackWindow()) {
1246
- return retryFn();
1247
- }
1248
- this._openFinitenessFallbackWindow(opts, reasonLabel, tokenBudget, rollbackSeqLen);
1249
- try {
1250
- return await retryFn();
1251
- } catch (error) {
1252
- this._closeFinitenessFallbackWindow(opts);
1253
- throw error;
1254
- }
475
+ return _retryWithPersistentFinitenessFallbackImpl.apply(this, arguments);
1255
476
  }
1256
477
 
1257
478
  async _retryDecodeStepWithFinitenessWindow(generatedIds, opts, reasonLabel) {
1258
- const windowTokens = this._resolveDeferredRoundingWindowTokens();
1259
- if (windowTokens <= 1) {
1260
- return this._retryWithFinitenessFallback(
1261
- opts,
1262
- reasonLabel,
1263
- () => this._decodeStep(generatedIds, opts)
1264
- );
1265
- }
1266
-
1267
- this._openFinitenessFallbackWindow(opts, reasonLabel, windowTokens);
1268
- try {
1269
- return await this._decodeStep(generatedIds, opts);
1270
- } catch (error) {
1271
- this._closeFinitenessFallbackWindow(opts);
1272
- throw error;
1273
- }
479
+ return _retryDecodeStepWithFinitenessWindowImpl.apply(this, arguments);
1274
480
  }
1275
481
 
1276
482
  // ==========================================================================
@@ -1287,591 +493,36 @@ export class PipelineGenerator {
1287
493
  * Only valid when no decode is in progress.
1288
494
  */
1289
495
  resetToSeqLen(seqLen) {
1290
- if (this.#state.isGenerating) {
496
+ if (this._state.isGenerating) {
1291
497
  throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
1292
498
  }
1293
499
  const target = Math.max(0, Math.floor(Number(seqLen) || 0));
1294
500
  if (!Number.isFinite(target)) {
1295
501
  throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
1296
502
  }
1297
- if (target > this.#state.currentSeqLen) {
503
+ if (target > this._state.currentSeqLen) {
1298
504
  throw new Error(
1299
- `InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
505
+ `InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this._state.currentSeqLen}`
1300
506
  );
1301
507
  }
1302
- this.#state.kvCache?.truncate?.(target);
1303
- this.#state.currentSeqLen = target;
508
+ this._state.kvCache?.truncate?.(target);
509
+ this._state.currentSeqLen = target;
1304
510
  }
1305
511
 
1306
512
  async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
1307
- const config = this.#state.modelConfig;
1308
- const canvasLength = canvasIds.length;
1309
- const hiddenSize = config.hiddenSize;
1310
- const vocabSize = config.vocabSize;
1311
- const elementCount = canvasLength * hiddenSize;
1312
- const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
1313
- selfConditioningLogits,
1314
- canvasLength,
1315
- hiddenSize
1316
- );
1317
- const logitsState = softEmbeddingState
1318
- ? null
1319
- : normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
1320
- const logits = (softEmbeddingState || logitsState)
1321
- ? null
1322
- : normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
1323
- const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
1324
- if (!weights || typeof weights !== 'object') {
1325
- throw new Error(
1326
- 'DiffusionGemma self-conditioning weights were not loaded. ' +
1327
- 'Expected model.decoder.self_conditioning tensors in the manifest.'
1328
- );
1329
- }
1330
-
1331
- const embedBufferRaw = this.#state.weights.get('embed');
1332
- if (!embedBufferRaw) {
1333
- throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
1334
- }
1335
- const borrowed = {
1336
- preNorm: null,
1337
- gateProj: null,
1338
- upProj: null,
1339
- downProj: null,
1340
- softEmbedding: null,
1341
- };
1342
- let softLogitsTensor = null;
1343
- let softLogitsOwned = true;
1344
- let softEmbeddingStateTensor = null;
1345
- let softEmbeddingStateOwned = false;
1346
- let softmaxTensor = null;
1347
- let softEmbeddings = null;
1348
- let scaledSoftEmbeddings = null;
1349
- let baseEmbeddings = null;
1350
- let normedSoft = null;
1351
- let gate = null;
1352
- let up = null;
1353
- let activated = null;
1354
- let down = null;
1355
- let combined = null;
1356
- let output = null;
1357
-
1358
- const releaseTensorOnce = (() => {
1359
- const released = new Set();
1360
- return (tensor) => {
1361
- const buffer = tensor?.buffer ?? null;
1362
- if (!buffer || released.has(buffer)) return;
1363
- released.add(buffer);
1364
- releaseBuffer(buffer);
1365
- };
1366
- })();
1367
-
1368
- try {
1369
- borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
1370
- borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
1371
- borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
1372
- borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
1373
- if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
1374
- throw new Error(
1375
- 'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
1376
- );
1377
- }
1378
- borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
1379
- ? null
1380
- : borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1381
-
1382
- const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1383
- const embedDtype = isCpuWeightBuffer(embedBufferRaw)
1384
- ? embedBufferRaw.dtype
1385
- : getWeightDtype(embedBufferRaw);
1386
- const embedMetadata = getWeightMetadata(embedBufferRaw);
1387
- baseEmbeddings = await embed(canvasIds, embedBuffer, {
1388
- hiddenSize,
1389
- vocabSize,
1390
- scaleEmbeddings: config.scaleEmbeddings,
1391
- embeddingScale: config.embeddingScale,
1392
- debug: opts.debug,
1393
- recorder: null,
1394
- transpose: this.#state.embeddingTranspose,
1395
- activationDtype: 'f32',
1396
- embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
1397
- embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1398
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1399
- operatorDiagnostics: this.#state.operatorDiagnostics,
1400
- });
1401
-
1402
- if (softEmbeddingState) {
1403
- softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
1404
- softEmbeddings = createTensor(
1405
- softEmbeddingState.buffer,
1406
- softEmbeddingState.dtype,
1407
- [canvasLength, hiddenSize],
1408
- 'diffusion_gemma_self_conditioning_soft_embedding'
1409
- );
1410
- softEmbeddingStateTensor = softEmbeddings;
1411
- if (!softEmbeddingState.scaled) {
1412
- scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1413
- count: elementCount,
1414
- });
1415
- if (softEmbeddingStateOwned) {
1416
- releaseTensorOnce(softEmbeddings);
1417
- }
1418
- softEmbeddings = scaledSoftEmbeddings;
1419
- softEmbeddingStateTensor = null;
1420
- scaledSoftEmbeddings = null;
1421
- }
1422
- } else if (logitsState || logits) {
1423
- const softmaxTemperature = logitsState?.temperature ?? 1.0;
1424
- if (logitsState) {
1425
- softLogitsOwned = logitsState.releaseOnUse;
1426
- softLogitsTensor = createTensor(
1427
- logitsState.logitsBuffer,
1428
- logitsState.logitsDtype,
1429
- [canvasLength, vocabSize],
1430
- 'diffusion_gemma_self_conditioning_logits'
1431
- );
1432
- } else {
1433
- const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
1434
- uploadData(logitsBuffer, logits);
1435
- softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
1436
- }
1437
- if (canUseChunkedSoftEmbeddingLogits(
1438
- logitsState,
1439
- borrowed.softEmbedding?.value,
1440
- this.#state.embeddingTranspose
1441
- )) {
1442
- softEmbeddings = await runSoftEmbeddingLogitsF16(
1443
- softLogitsTensor,
1444
- borrowed.softEmbedding.value,
1445
- canvasLength,
1446
- hiddenSize,
1447
- vocabSize,
1448
- {
1449
- temperature: softmaxTemperature,
1450
- chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1451
- }
1452
- );
1453
- } else {
1454
- softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
1455
- batchSize: canvasLength,
1456
- size: vocabSize,
1457
- temperature: softmaxTemperature,
1458
- });
1459
- softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
1460
- ? await runSoftEmbeddingSplitF16(
1461
- softmaxTensor,
1462
- embedBufferRaw,
1463
- canvasLength,
1464
- hiddenSize,
1465
- vocabSize
1466
- )
1467
- : await runMatmul(
1468
- softmaxTensor,
1469
- borrowed.softEmbedding.value,
1470
- canvasLength,
1471
- hiddenSize,
1472
- vocabSize,
1473
- {
1474
- transposeB: this.#state.embeddingTranspose === true,
1475
- role: 'diffusion_gemma_self_conditioning_embed',
1476
- outputDtype: 'f32',
1477
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1478
- }
1479
- );
1480
- }
1481
- scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1482
- count: elementCount,
1483
- });
1484
- if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1485
- releaseTensorOnce(softEmbeddings);
1486
- }
1487
- softEmbeddings = scaledSoftEmbeddings;
1488
- scaledSoftEmbeddings = null;
1489
- } else {
1490
- const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
1491
- uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
1492
- softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
1493
- }
1494
-
1495
- normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
1496
- batchSize: canvasLength,
1497
- hiddenSize,
1498
- rmsNormWeightOffset: false,
1499
- });
1500
- const intermediateSize = config.intermediateSize;
1501
- gate = await runMatmul(
1502
- normedSoft,
1503
- borrowed.gateProj.value,
1504
- canvasLength,
1505
- intermediateSize,
1506
- hiddenSize,
1507
- {
1508
- transposeB: 'auto',
1509
- role: 'diffusion_gemma_self_conditioning_gate',
1510
- outputDtype: 'f32',
1511
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1512
- }
1513
- );
1514
- up = await runMatmul(
1515
- normedSoft,
1516
- borrowed.upProj.value,
1517
- canvasLength,
1518
- intermediateSize,
1519
- hiddenSize,
1520
- {
1521
- transposeB: 'auto',
1522
- role: 'diffusion_gemma_self_conditioning_up',
1523
- outputDtype: 'f32',
1524
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1525
- }
1526
- );
1527
- activated = await runGeLU(up, {
1528
- size: canvasLength * intermediateSize,
1529
- gate,
1530
- });
1531
- down = await runMatmul(
1532
- activated,
1533
- borrowed.downProj.value,
1534
- canvasLength,
1535
- hiddenSize,
1536
- intermediateSize,
1537
- {
1538
- transposeB: 'auto',
1539
- role: 'diffusion_gemma_self_conditioning_down',
1540
- outputDtype: 'f32',
1541
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1542
- }
1543
- );
1544
- combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
1545
- executionPolicies: this.#state.executionV1State?.policies ?? null,
1546
- });
1547
- const postNormWeight = weights.postNorm
1548
- ? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
1549
- : {
1550
- value: getQKNormOnesBuffer(hiddenSize),
1551
- owned: false,
1552
- };
1553
- borrowed.postNorm = postNormWeight;
1554
- output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
1555
- batchSize: canvasLength,
1556
- hiddenSize,
1557
- rmsNormWeightOffset: false,
1558
- });
1559
- return output;
1560
- } catch (error) {
1561
- releaseTensorOnce(output);
1562
- throw error;
1563
- } finally {
1564
- releaseTensorOnce(combined);
1565
- releaseTensorOnce(down);
1566
- releaseTensorOnce(activated);
1567
- releaseTensorOnce(up);
1568
- releaseTensorOnce(gate);
1569
- releaseTensorOnce(normedSoft);
1570
- if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1571
- releaseTensorOnce(softEmbeddings);
1572
- }
1573
- releaseTensorOnce(softmaxTensor);
1574
- if (softLogitsOwned) {
1575
- releaseTensorOnce(softLogitsTensor);
1576
- }
1577
- releaseTensorOnce(baseEmbeddings);
1578
- releaseBorrowedWeight(borrowed.postNorm);
1579
- releaseBorrowedWeight(borrowed.downProj);
1580
- releaseBorrowedWeight(borrowed.upProj);
1581
- releaseBorrowedWeight(borrowed.gateProj);
1582
- releaseBorrowedWeight(borrowed.preNorm);
1583
- releaseBorrowedWeight(borrowed.softEmbedding);
1584
- }
513
+ return _createDiffusionGemmaSelfConditioningEmbeddingsImpl.apply(this, arguments);
1585
514
  }
1586
515
 
1587
516
  async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
1588
- const embedBufferRaw = this.#state.weights.get('embed');
1589
- if (!canUseChunkedSoftEmbeddingLogits(
1590
- logitsState,
1591
- embedBufferRaw,
1592
- this.#state.embeddingTranspose
1593
- )) {
1594
- return null;
1595
- }
1596
-
1597
- const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1598
- let softEmbedding = null;
1599
- let scaledSoftEmbedding = null;
1600
- try {
1601
- const logitsTensor = createTensor(
1602
- logitsState.logitsBuffer,
1603
- logitsState.logitsDtype,
1604
- [canvasLength, vocabSize],
1605
- 'diffusion_gemma_self_conditioning_logits'
1606
- );
1607
- softEmbedding = await runSoftEmbeddingLogitsF16(
1608
- logitsTensor,
1609
- borrowed.value,
1610
- canvasLength,
1611
- hiddenSize,
1612
- vocabSize,
1613
- {
1614
- temperature: logitsState.temperature,
1615
- chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1616
- }
1617
- );
1618
- scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
1619
- count: canvasLength * hiddenSize,
1620
- });
1621
- const returnedBuffer = scaledSoftEmbedding.buffer;
1622
- return {
1623
- kind: 'soft_embedding',
1624
- buffer: returnedBuffer,
1625
- dtype: scaledSoftEmbedding.dtype,
1626
- canvasLength,
1627
- hiddenSize,
1628
- scaled: true,
1629
- releaseOnUse: true,
1630
- release() {
1631
- releaseBuffer(returnedBuffer);
1632
- },
1633
- };
1634
- } finally {
1635
- if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
1636
- releaseBuffer(softEmbedding.buffer);
1637
- }
1638
- releaseBorrowedWeight(borrowed);
1639
- }
517
+ return _createDiffusionGemmaSelfConditioningSoftEmbeddingStateImpl.apply(this, arguments);
1640
518
  }
1641
519
 
1642
520
  async computeDiffusionGemmaCanvasLogits(args, options = {}) {
1643
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
1644
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
1645
- throw new Error('Generation already in progress');
1646
- }
1647
- const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
1648
- if (canvasIds.length === 0) {
1649
- throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
1650
- }
1651
- this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
1652
- const seqLenBefore = this.#state.currentSeqLen;
1653
- const opts = {
1654
- ...resolvePrefillOptions(this.#state, {
1655
- ...options,
1656
- useChatTemplate: false,
1657
- }),
1658
- _diffusionGemmaDecoder: true,
1659
- };
1660
- let selfConditioned = null;
1661
- let currentHiddenBuffer = null;
1662
- try {
1663
- selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1664
- canvasIds,
1665
- args?.selfConditioningLogits ?? null,
1666
- opts
1667
- );
1668
- const prefillResult = await this._prefillToHidden(canvasIds, {
1669
- ...opts,
1670
- embeddingOverrides: {
1671
- offset: 0,
1672
- prefixLength: canvasIds.length,
1673
- embeddings: selfConditioned.buffer,
1674
- },
1675
- });
1676
- const {
1677
- numTokens,
1678
- currentRecorder,
1679
- recordProfile,
1680
- debugCheckBuffer,
1681
- } = prefillResult;
1682
- currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1683
- if (currentRecorder) {
1684
- recordPrefillRecorderStats(this.#state, currentRecorder);
1685
- await currentRecorder.submitAndWait();
1686
- await recordProfile(currentRecorder);
1687
- } else {
1688
- const device = getDevice();
1689
- if (device) {
1690
- await device.queue.onSubmittedWorkDone();
1691
- }
1692
- }
1693
- const logits = await computeLogits(
1694
- currentHiddenBuffer,
1695
- numTokens,
1696
- getLogitsWeights(this.#state),
1697
- getLogitsConfig(this.#state),
1698
- this.#state.useGPU,
1699
- this.#state.debugFlags,
1700
- undefined,
1701
- debugCheckBuffer,
1702
- this.#state.runtimeConfig.shared.debug.probes,
1703
- { lastPositionOnly: false },
1704
- this.#state.operatorDiagnostics
1705
- );
1706
- const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
1707
- if (logits.length !== expected) {
1708
- throw new Error(
1709
- `[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
1710
- );
1711
- }
1712
- return logits;
1713
- } finally {
1714
- this.#state.currentSeqLen = seqLenBefore;
1715
- if (currentHiddenBuffer) {
1716
- releaseBuffer(currentHiddenBuffer);
1717
- }
1718
- if (selfConditioned?.buffer) {
1719
- releaseBuffer(selfConditioned.buffer);
1720
- }
1721
- }
521
+ return computeDiffusionGemmaCanvasLogitsImpl.apply(this, arguments);
1722
522
  }
1723
523
 
1724
524
  async computeDiffusionGemmaCanvasStep(args, options = {}) {
1725
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
1726
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
1727
- throw new Error('Generation already in progress');
1728
- }
1729
- const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
1730
- if (canvasIds.length === 0) {
1731
- throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
1732
- }
1733
- this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
1734
- const temperature = args?.temperature;
1735
- if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
1736
- throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
1737
- }
1738
- const seqLenBefore = this.#state.currentSeqLen;
1739
- const opts = {
1740
- ...resolvePrefillOptions(this.#state, {
1741
- ...options,
1742
- useChatTemplate: false,
1743
- }),
1744
- _diffusionGemmaDecoder: true,
1745
- };
1746
- let selfConditioned = null;
1747
- let currentHiddenBuffer = null;
1748
- let logitsBuffer = null;
1749
- let statsBuffers = null;
1750
- try {
1751
- selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1752
- canvasIds,
1753
- args?.selfConditioningLogits ?? null,
1754
- opts
1755
- );
1756
- const prefillResult = await this._prefillToHidden(canvasIds, {
1757
- ...opts,
1758
- embeddingOverrides: {
1759
- offset: 0,
1760
- prefixLength: canvasIds.length,
1761
- embeddings: selfConditioned.buffer,
1762
- },
1763
- });
1764
- const {
1765
- numTokens,
1766
- currentRecorder,
1767
- recordProfile,
1768
- debugCheckBuffer,
1769
- } = prefillResult;
1770
- currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1771
- if (currentRecorder) {
1772
- recordPrefillRecorderStats(this.#state, currentRecorder);
1773
- await currentRecorder.submitAndWait();
1774
- await recordProfile(currentRecorder);
1775
- } else {
1776
- const device = getDevice();
1777
- if (device) {
1778
- await device.queue.onSubmittedWorkDone();
1779
- }
1780
- }
1781
- if (selfConditioned?.buffer) {
1782
- releaseBuffer(selfConditioned.buffer);
1783
- selfConditioned = null;
1784
- }
1785
-
1786
- const logitsResult = await computeLogitsGPU(
1787
- currentHiddenBuffer,
1788
- numTokens,
1789
- getLogitsWeights(this.#state),
1790
- getLogitsConfig(this.#state),
1791
- this.#state.debugFlags,
1792
- this.#state.operatorDiagnostics
1793
- );
1794
- if (!logitsResult?.logitsBuffer) {
1795
- throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
1796
- }
1797
- logitsBuffer = logitsResult.logitsBuffer;
1798
- if (currentHiddenBuffer) {
1799
- releaseBuffer(currentHiddenBuffer);
1800
- currentHiddenBuffer = null;
1801
- }
1802
- if (logitsResult.logitsDtype !== 'f32') {
1803
- throw new Error(
1804
- `[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
1805
- );
1806
- }
1807
- if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
1808
- throw new Error(
1809
- `[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
1810
- `got ${logitsResult.vocabSize}.`
1811
- );
1812
- }
1813
-
1814
- statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
1815
- canvasLength: canvasIds.length,
1816
- vocabSize: logitsResult.vocabSize,
1817
- temperature,
1818
- padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
1819
- logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
1820
- });
1821
- const [argmaxData, entropyData] = await Promise.all([
1822
- readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
1823
- readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
1824
- ]);
1825
- const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
1826
- const entropies = new Float32Array(entropyData);
1827
- const logitsState = {
1828
- logitsBuffer,
1829
- logitsDtype: logitsResult.logitsDtype,
1830
- vocabSize: logitsResult.vocabSize,
1831
- canvasLength: canvasIds.length,
1832
- temperature,
1833
- releaseOnUse: true,
1834
- };
1835
- const selfConditioningState =
1836
- await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
1837
- logitsState,
1838
- canvasIds.length,
1839
- this.#state.modelConfig.hiddenSize,
1840
- logitsResult.vocabSize
1841
- );
1842
- if (selfConditioningState) {
1843
- releaseBuffer(logitsBuffer);
1844
- logitsBuffer = null;
1845
- } else {
1846
- const returnedLogitsBuffer = logitsBuffer;
1847
- logitsBuffer = null;
1848
- logitsState.release = () => {
1849
- releaseBuffer(returnedLogitsBuffer);
1850
- };
1851
- }
1852
- return {
1853
- argmaxCanvas,
1854
- entropies,
1855
- selfConditioningLogits: selfConditioningState ?? logitsState,
1856
- };
1857
- } finally {
1858
- this.#state.currentSeqLen = seqLenBefore;
1859
- if (statsBuffers?.argmaxBuffer) {
1860
- releaseBuffer(statsBuffers.argmaxBuffer);
1861
- }
1862
- if (statsBuffers?.entropyBuffer) {
1863
- releaseBuffer(statsBuffers.entropyBuffer);
1864
- }
1865
- if (logitsBuffer) {
1866
- releaseBuffer(logitsBuffer);
1867
- }
1868
- if (currentHiddenBuffer) {
1869
- releaseBuffer(currentHiddenBuffer);
1870
- }
1871
- if (selfConditioned?.buffer) {
1872
- releaseBuffer(selfConditioned.buffer);
1873
- }
1874
- }
525
+ return computeDiffusionGemmaCanvasStepImpl.apply(this, arguments);
1875
526
  }
1876
527
 
1877
528
  async *generate(prompt, options = {}) {
@@ -1883,873 +534,35 @@ export class PipelineGenerator {
1883
534
  }
1884
535
 
1885
536
  async generateTokenIds(prompt, options = {}) {
1886
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
1887
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
1888
-
1889
- validateCallTimeOptions(options);
1890
-
1891
- this.#state.isGenerating = true;
1892
- this._resetDecodeRuntimeState();
1893
- this.#state.stats.gpuTimePrefillMs = undefined;
1894
- this.#state.stats.gpuTimeDecodeMs = undefined;
1895
- this.#state.stats.decodeRecordMs = 0;
1896
- this.#state.stats.decodeRecordOps = 0;
1897
- this.#state.stats.decodeRecordPasses = 0;
1898
- this.#state.stats.decodeRecordOpLabels = {};
1899
- this.#state.stats.decodeSubmitWaitMs = 0;
1900
- this.#state.stats.decodeReadbackWaitMs = 0;
1901
- this.#state.stats.decodeReadbackMapWaitMs = 0;
1902
- this.#state.stats.decodeReadbackCleanupMs = 0;
1903
- this.#state.stats.decodeReadbackCopyMs = 0;
1904
- this.#state.stats.prefillRecordMs = 0;
1905
- this.#state.stats.prefillRecordOps = 0;
1906
- this.#state.stats.prefillRecordPasses = 0;
1907
- this.#state.stats.prefillRecordOpLabels = {};
1908
- this.#state.stats.prefillSubmitWaitMs = 0;
1909
- this.#state.stats.singleTokenSubmitWaitMs = 0;
1910
- this.#state.stats.singleTokenReadbackWaitMs = 0;
1911
- this.#state.stats.singleTokenReadbackMapWaitMs = 0;
1912
- this.#state.stats.singleTokenReadbackCleanupMs = 0;
1913
- this.#state.stats.singleTokenReadbackCopyMs = 0;
1914
- this.#state.stats.singleTokenOrchestrationMs = 0;
1915
- this.#state.stats.pleHotVocabularyHits = 0;
1916
- this.#state.stats.pleHotVocabularyMisses = 0;
1917
- this.#state.stats.plePreparedTokenCacheHits = 0;
1918
- this.#state.stats.plePreparedTokenCacheMisses = 0;
1919
- this.#state.stats.plePreparedTokenCacheEntries = 0;
1920
- this.#state.stats.plePreparedTokenCacheBytes = 0;
1921
- this.#state.stats.decodeMode = null;
1922
- this.#state.stats.batchGuardReason = null;
1923
- this.#state.stats.stopReason = null;
1924
- this.#state.stats.stopTokenId = null;
1925
- this.#state.stats.ttftMs = 0;
1926
- const startTime = performance.now();
1927
- const opts = resolveGenerateOptions(this.#state, options);
1928
- opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
1929
- // Validate and normalize sampling parameters through single source of truth
1930
- const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
1931
- opts.temperature = samplingConfig.temperature;
1932
- opts.topP = samplingConfig.topP;
1933
- opts.topK = samplingConfig.topK;
1934
- opts.repetitionPenalty = samplingConfig.repetitionPenalty;
1935
- opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
1936
- const diagnosticsEnabled = options?.diagnostics?.enabled === true
1937
- || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
1938
- const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
1939
- if (diagnosticsEnabled || tsirFixtureCfg) {
1940
- const captureConfig = {
1941
- ...createDefaultCaptureConfig(),
1942
- enabled: true,
1943
- defaultLevel: CAPTURE_LEVELS.SLICE,
1944
- ...(options?.diagnostics?.captureConfig ?? {}),
1945
- };
1946
- validateCaptureConfig(captureConfig);
1947
- this.#state.operatorDiagnostics = {
1948
- enabled: diagnosticsEnabled === true,
1949
- captureConfig,
1950
- emitter: diagnosticsEnabled ? new OperatorEventEmitter({
1951
- modelHash: this.#state.manifest?.modelId ?? null,
1952
- runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
1953
- executionPlanHash: opts.executionPlan?.id ?? null,
1954
- }) : null,
1955
- tsirFixture: tsirFixtureCfg ? {
1956
- dir: tsirFixtureCfg.dir,
1957
- layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
1958
- ? tsirFixtureCfg.layerFilter
1959
- : null,
1960
- records: [],
1961
- } : null,
1962
- };
1963
- }
1964
-
1965
- try {
1966
- const prefillStartSeqLen = this.#state.currentSeqLen;
1967
- const prefillStart = performance.now();
1968
- const { inputIds, logits: initialPrefillLogits } = await this._prefillPromptToLogits(prompt, opts, 'generateTokenIds');
1969
- let prefillLogits = initialPrefillLogits;
1970
- this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
1971
- this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
1972
- const generatedIds = [...inputIds];
1973
- this.#state.stats.prefillTokens = inputIds.length;
1974
-
1975
- let firstToken;
1976
- try {
1977
- firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
1978
- } catch (error) {
1979
- if (!this._shouldUseFinitenessFallback(error, 'prefill-sample')) {
1980
- throw error;
1981
- }
1982
- prefillLogits = await this._retryWithPersistentFinitenessFallback(
1983
- opts,
1984
- 'prefill-sample',
1985
- opts.maxTokens,
1986
- () => this._prefill(inputIds, opts),
1987
- prefillStartSeqLen
1988
- );
1989
- firstToken = this._sampleNextTokenFromLogits(prefillLogits, generatedIds, opts);
1990
- }
1991
-
1992
- const stopTokenIds = this.#state.modelConfig.stopTokenIds;
1993
- const eosToken = this.#state.tokenizer.getSpecialTokens?.()?.eos;
1994
- const stopSequenceStart = inputIds.length;
1995
- generatedIds.push(firstToken);
1996
- const tokenIds = [firstToken];
1997
- this.#state.stats.ttftMs = performance.now() - startTime;
1998
- markKernelCacheWarmed();
1999
-
2000
- const decodeRuntime = {
2001
- stopTokenIds,
2002
- eosToken,
2003
- stopSequenceStart,
2004
- decodeToken: () => '',
2005
- emitMode: 'token',
2006
- };
2007
-
2008
- if (!this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
2009
- for await (const tokenId of this._runDecodeLoop(generatedIds, opts, options, decodeRuntime)) {
2010
- tokenIds.push(tokenId);
2011
- }
2012
- } else {
2013
- this._recordStopReason('stop-token-or-sequence', firstToken);
2014
- this.#state.stats.decodeTimeMs = 0;
2015
- this.#state.stats.tokensGenerated = 1;
2016
- this.#state.stats.decodeTokens = 1;
2017
- }
2018
-
2019
- this.#state.stats.totalTimeMs = performance.now() - startTime;
2020
-
2021
- return {
2022
- tokenIds,
2023
- stats: this.#state.stats,
2024
- };
2025
- } finally {
2026
- this._closeFinitenessFallbackWindow(opts);
2027
- if (this.#state.stats.executionPlan) {
2028
- this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
2029
- }
2030
- resetActiveExecutionPlan(this.#state);
2031
- this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2032
- ? {
2033
- enabled: true,
2034
- timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2035
- recordCount: this.#state.operatorDiagnostics.emitter.length,
2036
- }
2037
- : null;
2038
- this.#state.operatorDiagnostics = null;
2039
- this.#state.isGenerating = false;
2040
- }
537
+ return generateTokenIdsImpl.apply(this, arguments);
2041
538
  }
2042
539
 
2043
-
2044
540
  async prefillKVOnly(prompt, options = {}) {
2045
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2046
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
2047
- throw new Error('Generation already in progress');
2048
- }
2049
- assertIncrementalDecodeSupport(this.#state, 'prefillKVOnly');
2050
- this._resetDecodeRuntimeState();
2051
- this.#state.stats.gpuTimePrefillMs = undefined;
2052
- this.#state.stats.prefillProfileSteps = [];
2053
- const opts = resolvePrefillOptions(this.#state, options);
2054
- const prefillStartSeqLen = this.#state.currentSeqLen;
2055
- const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillKVOnly', opts.inputIds);
2056
- if (opts.debug) {
2057
- log.debug('Pipeline', `PrefillKVOnly: ${inputIds.length} tokens`);
2058
- }
2059
-
2060
- try {
2061
- let prefillResult;
2062
- try {
2063
- prefillResult = await this._prefillToHidden(inputIds, opts);
2064
- } catch (error) {
2065
- if (this._shouldUseFinitenessFallback(error, 'prefillKVOnly')) {
2066
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillKVOnly. Retrying with F32 precision.`);
2067
- prefillResult = await this._retryWithPersistentFinitenessFallback(
2068
- opts,
2069
- 'prefillKVOnly',
2070
- 1,
2071
- () => this._prefillToHidden(inputIds, opts),
2072
- prefillStartSeqLen
2073
- );
2074
- } else {
2075
- throw error;
2076
- }
2077
- }
2078
-
2079
- const {
2080
- numTokens,
2081
- startPos,
2082
- currentRecorder,
2083
- recordProfile,
2084
- currentHiddenBuffer,
2085
- } = prefillResult;
2086
-
2087
- // Ensure prefill work completes before returning a usable snapshot.
2088
- if (currentRecorder) {
2089
- recordPrefillRecorderStats(this.#state, currentRecorder);
2090
- await currentRecorder.submitAndWait();
2091
- await recordProfile(currentRecorder);
2092
- } else {
2093
- const device = getDevice();
2094
- if (device) {
2095
- await device.queue.onSubmittedWorkDone();
2096
- }
2097
- }
2098
-
2099
- this.#state.currentSeqLen = startPos + numTokens;
2100
- releaseBuffer(currentHiddenBuffer);
2101
-
2102
- const snapshot = this.#state.kvCache?.clone();
2103
- if (!snapshot) {
2104
- throw new Error('KV cache unavailable after prefill');
2105
- }
2106
-
2107
- return {
2108
- cache: snapshot,
2109
- seqLen: this.#state.currentSeqLen,
2110
- tokens: inputIds,
2111
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2112
- };
2113
- } finally {
2114
- this._closeFinitenessFallbackWindow(opts);
2115
- }
541
+ return prefillKVOnlyImpl.apply(this, arguments);
2116
542
  }
2117
543
 
2118
544
  async prefillForLoRATraining(inputIds, options = {}) {
2119
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2120
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
2121
- if (!Array.isArray(inputIds) && !(inputIds instanceof Int32Array) && !(inputIds instanceof Uint32Array)) {
2122
- throw new Error('prefillForLoRATraining requires token IDs.');
2123
- }
2124
- const tokens = Array.from(inputIds, (value) => Number(value));
2125
- if (tokens.length === 0) throw new Error('prefillForLoRATraining requires at least one token.');
2126
- this._assertTokenIdsInRange(tokens, 'prefillForLoRATraining.inputIds');
2127
-
2128
- const config = this.#state.modelConfig;
2129
- const layerIdx = Number(options.layerIdx);
2130
- if (!Number.isInteger(layerIdx) || layerIdx !== config.numLayers - 1) {
2131
- throw new Error(`native_lora_target_not_supported: layerIdx must be the final layer (${config.numLayers - 1}).`);
2132
- }
2133
- if (options.module !== 'down_proj') {
2134
- throw new Error('native_lora_target_not_supported: module must be down_proj.');
2135
- }
2136
- if (config.layerTypes?.[layerIdx] !== 'full_attention') {
2137
- throw new Error('native_lora_target_not_supported: final layer must be full_attention.');
2138
- }
2139
- if (config.postFeedforwardNorm === true) {
2140
- throw new Error('native_lora_target_not_supported: final-layer post-feedforward normalization is not supported.');
2141
- }
2142
- if (this.#state.lora) {
2143
- throw new Error('native_lora_base_capture_requires_inactive_adapter.');
2144
- }
2145
-
2146
- this.resetToSeqLen(0);
2147
- this._resetDecodeRuntimeState();
2148
- const opts = resolvePrefillOptions(this.#state, {
2149
- ...options,
2150
- inputIds: tokens,
2151
- useChatTemplate: false,
2152
- });
2153
- let activationTensor = null;
2154
- let hiddenTensor = null;
2155
- let disposed = false;
2156
- const releaseOwned = () => {
2157
- if (disposed) return;
2158
- disposed = true;
2159
- if (activationTensor?.buffer) releaseBuffer(activationTensor.buffer);
2160
- if (hiddenTensor?.buffer) releaseBuffer(hiddenTensor.buffer);
2161
- };
2162
- const capture = async ({ tensor, numTokens, hiddenSize, recorder }) => {
2163
- if (activationTensor) {
2164
- throw new Error('native_lora_training_capture_duplicated.');
2165
- }
2166
- const bytesPerElement = tensor.dtype === 'f16' ? 2 : 4;
2167
- const byteLength = Math.ceil((numTokens * hiddenSize * bytesPerElement) / 4) * 4;
2168
- const buffer = acquireBuffer(byteLength, undefined, `L${layerIdx}.native_lora_ffn_act`);
2169
- if (recorder) {
2170
- recorder.getEncoder().copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
2171
- } else {
2172
- const device = getDevice();
2173
- const encoder = device.createCommandEncoder({ label: `L${layerIdx}.native_lora_capture` });
2174
- encoder.copyBufferToBuffer(tensor.buffer, 0, buffer, 0, byteLength);
2175
- device.queue.submit([encoder.finish()]);
2176
- }
2177
- activationTensor = createTensor(
2178
- buffer,
2179
- tensor.dtype,
2180
- [numTokens, hiddenSize],
2181
- `L${layerIdx}.native_lora_ffn_act`
2182
- );
2183
- };
2184
-
2185
- try {
2186
- const prefill = await this._prefillToHidden(tokens, {
2187
- ...opts,
2188
- _trainingCapture: { layerIdx, stage: 'ffn_act', capture },
2189
- });
2190
- if (prefill.currentRecorder) {
2191
- recordPrefillRecorderStats(this.#state, prefill.currentRecorder);
2192
- await prefill.currentRecorder.submitAndWait();
2193
- await prefill.recordProfile(prefill.currentRecorder);
2194
- } else {
2195
- await getDevice().queue.onSubmittedWorkDone();
2196
- }
2197
- if (!activationTensor) {
2198
- releaseBuffer(prefill.currentHiddenBuffer);
2199
- throw new Error('native_lora_training_activation_not_captured.');
2200
- }
2201
- hiddenTensor = createTensor(
2202
- prefill.currentHiddenBuffer,
2203
- prefill.activationDtype,
2204
- [prefill.numTokens, config.hiddenSize],
2205
- `L${layerIdx}.native_lora_base_hidden`
2206
- );
2207
- this.resetToSeqLen(0);
2208
- return {
2209
- inputIds: tokens,
2210
- layerIdx,
2211
- module: 'down_proj',
2212
- activation: activationTensor,
2213
- baseHidden: hiddenTensor,
2214
- dispose: releaseOwned,
2215
- };
2216
- } catch (error) {
2217
- releaseOwned();
2218
- this.resetToSeqLen(0);
2219
- throw error;
2220
- }
545
+ return prefillForLoRATrainingImpl.apply(this, arguments);
2221
546
  }
2222
547
 
2223
548
  async prefillWithEmbedding(prompt, options = {}) {
2224
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2225
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
2226
- throw new Error('Generation already in progress');
2227
- }
2228
- assertIncrementalDecodeSupport(this.#state, 'prefillWithEmbedding');
2229
- const totalStart = performance.now();
2230
- this._resetDecodeRuntimeState();
2231
- this.#state.stats.gpuTimePrefillMs = undefined;
2232
- this.#state.stats.prefillProfileSteps = [];
2233
- const opts = resolvePrefillEmbeddingOptions(this.#state, options);
2234
- const diagnosticsEnabled = options?.diagnostics?.enabled === true
2235
- || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2236
- const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
2237
- if (diagnosticsEnabled || tsirFixtureCfg) {
2238
- const captureConfig = {
2239
- ...createDefaultCaptureConfig(),
2240
- enabled: true,
2241
- defaultLevel: CAPTURE_LEVELS.SLICE,
2242
- ...(options?.diagnostics?.captureConfig ?? {}),
2243
- };
2244
- validateCaptureConfig(captureConfig);
2245
- this.#state.operatorDiagnostics = {
2246
- enabled: diagnosticsEnabled === true,
2247
- captureConfig,
2248
- emitter: diagnosticsEnabled ? new OperatorEventEmitter({
2249
- modelHash: this.#state.manifest?.modelId ?? null,
2250
- runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2251
- executionPlanHash: opts.executionPlan?.id ?? null,
2252
- }) : null,
2253
- tsirFixture: tsirFixtureCfg ? {
2254
- dir: tsirFixtureCfg.dir,
2255
- layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
2256
- ? tsirFixtureCfg.layerFilter
2257
- : null,
2258
- records: [],
2259
- } : null,
2260
- };
2261
- }
2262
- const prefillStartSeqLen = this.#state.currentSeqLen;
2263
- const inputStart = performance.now();
2264
- const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
2265
- const inputMs = performance.now() - inputStart;
2266
- if (opts.debug) {
2267
- log.debug('Pipeline', `PrefillWithEmbedding: ${inputIds.length} tokens (mode=${opts.embeddingMode})`);
2268
- }
2269
-
2270
- try {
2271
- let prefillResult;
2272
- const prefillStart = performance.now();
2273
- try {
2274
- prefillResult = await this._prefillToHidden(inputIds, {
2275
- ...opts,
2276
- _embeddingOnly: options.__skipStateSnapshot === true,
2277
- });
2278
- } catch (error) {
2279
- if (shouldRetryWithFinitenessFallback(error)) {
2280
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf during prefillWithEmbedding. Retrying with F32 precision.`);
2281
- prefillResult = await this._retryWithPersistentFinitenessFallback(
2282
- opts,
2283
- 'prefillWithEmbedding',
2284
- 1,
2285
- () => this._prefillToHidden(inputIds, {
2286
- ...opts,
2287
- _embeddingOnly: options.__skipStateSnapshot === true,
2288
- }),
2289
- prefillStartSeqLen
2290
- );
2291
- } else {
2292
- throw error;
2293
- }
2294
- }
2295
- const prefillMs = performance.now() - prefillStart;
2296
-
2297
- const {
2298
- numTokens,
2299
- config,
2300
- startPos,
2301
- activationDtype,
2302
- activationBytes,
2303
- currentRecorder,
2304
- recordProfile,
2305
- currentHiddenBuffer,
2306
- } = prefillResult;
2307
-
2308
- // Ensure prefill work completes before readback.
2309
- let submitWaitMs = 0;
2310
- const submitWaitStart = performance.now();
2311
- if (currentRecorder) {
2312
- recordPrefillRecorderStats(this.#state, currentRecorder);
2313
- await currentRecorder.submitAndWait();
2314
- await recordProfile(currentRecorder);
2315
- } else {
2316
- const device = getDevice();
2317
- if (device) {
2318
- await device.queue.onSubmittedWorkDone();
2319
- }
2320
- }
2321
- submitWaitMs = performance.now() - submitWaitStart;
2322
-
2323
- if (!allowReadback('pipeline.prefill.embedding')) {
2324
- throw new Error('GPU readback disabled; cannot return embedding');
2325
- }
2326
-
2327
- let embedding;
2328
- let tokenEmbeddings = null;
2329
- let logits = null;
2330
- let hiddenBytes = 0;
2331
- let readbackMs = 0;
2332
- let decodeHiddenMs = 0;
2333
- let finalNormMs = 0;
2334
- let extractMs = 0;
2335
- let logitsMs = 0;
2336
- try {
2337
- if (options.__returnSequenceLogits === true) {
2338
- const logitsStart = performance.now();
2339
- logits = await computeLogits(
2340
- currentHiddenBuffer,
2341
- numTokens,
2342
- getLogitsWeights(this.#state),
2343
- getLogitsConfig(this.#state),
2344
- this.#state.useGPU,
2345
- this.#state.debugFlags,
2346
- undefined,
2347
- undefined,
2348
- this.#state.runtimeConfig.shared.debug.probes,
2349
- { lastPositionOnly: false },
2350
- this.#state.operatorDiagnostics
2351
- );
2352
- logitsMs = performance.now() - logitsStart;
2353
- }
2354
- const hiddenSize = config.hiddenSize;
2355
- hiddenBytes = numTokens * hiddenSize * activationBytes;
2356
- const readbackStart = performance.now();
2357
- const hiddenData = await readBuffer(currentHiddenBuffer, hiddenBytes);
2358
- readbackMs = performance.now() - readbackStart;
2359
- if (hiddenData.byteLength === 0) {
2360
- throw new Error('GPU readback disabled; cannot return embedding');
2361
- }
2362
- const decodeStart = performance.now();
2363
- const hiddenStates = decodeReadback(hiddenData, activationDtype);
2364
- decodeHiddenMs = performance.now() - decodeStart;
2365
- const finalNormStart = performance.now();
2366
- const finalNormWeights = await this._getFinalNormWeights();
2367
- finalNormMs = performance.now() - finalNormStart;
2368
- const extractStart = performance.now();
2369
- if (options.__returnTokenEmbeddings === true) {
2370
- tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
2371
- hiddenStates,
2372
- numTokens,
2373
- hiddenSize,
2374
- finalNormWeights,
2375
- config
2376
- );
2377
- }
2378
- embedding = extractEmbeddingFromHidden(
2379
- hiddenStates,
2380
- numTokens,
2381
- hiddenSize,
2382
- opts.embeddingMode,
2383
- finalNormWeights,
2384
- config,
2385
- this.#state.embeddingPostprocessor,
2386
- tokenEmbeddings,
2387
- this.#state.weights.get('final_norm_bias') ?? null
2388
- );
2389
- extractMs = performance.now() - extractStart;
2390
- } finally {
2391
- releaseBuffer(currentHiddenBuffer);
2392
- }
2393
-
2394
- this.#state.currentSeqLen = startPos + numTokens;
2395
- const phase = {
2396
- totalMs: performance.now() - totalStart,
2397
- inputMs,
2398
- prefillMs,
2399
- submitWaitMs,
2400
- readbackMs,
2401
- decodeHiddenMs,
2402
- finalNormMs,
2403
- extractMs,
2404
- logitsMs,
2405
- hiddenBytes,
2406
- tokens: numTokens,
2407
- activationDtype,
2408
- prefillRecordMs: this.#state.stats.prefillRecordMs ?? null,
2409
- prefillRecordOps: this.#state.stats.prefillRecordOps ?? null,
2410
- prefillRecordPasses: this.#state.stats.prefillRecordPasses ?? null,
2411
- prefillRecordOpLabels: this.#state.stats.prefillRecordOpLabels ?? null,
2412
- prefillSubmitWaitMs: this.#state.stats.prefillSubmitWaitMs ?? null,
2413
- gpuPrefillMs: this.#state.stats.gpuTimePrefillMs ?? null,
2414
- };
2415
-
2416
- // Batch embedding skips expensive KV cache clone and linear attention clone
2417
- // since the caller will reset immediately after extracting the embedding.
2418
- if (options.__skipStateSnapshot) {
2419
- return {
2420
- cache: null,
2421
- seqLen: this.#state.currentSeqLen,
2422
- tokens: inputIds,
2423
- embedding,
2424
- tokenEmbeddings,
2425
- logits,
2426
- embeddingMode: opts.embeddingMode,
2427
- phase,
2428
- linearAttention: null,
2429
- };
2430
- }
2431
-
2432
- const snapshot = this.#state.kvCache?.clone();
2433
- if (!snapshot) {
2434
- throw new Error('KV cache unavailable after prefill');
2435
- }
2436
-
2437
- return {
2438
- cache: snapshot,
2439
- seqLen: this.#state.currentSeqLen,
2440
- tokens: inputIds,
2441
- embedding,
2442
- tokenEmbeddings,
2443
- logits,
2444
- embeddingMode: opts.embeddingMode,
2445
- phase,
2446
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2447
- };
2448
- } finally {
2449
- this._closeFinitenessFallbackWindow(opts);
2450
- this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2451
- ? {
2452
- enabled: true,
2453
- timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2454
- recordCount: this.#state.operatorDiagnostics.emitter.length,
2455
- }
2456
- : null;
2457
- this.#state.operatorDiagnostics = null;
2458
- }
549
+ return prefillWithEmbeddingImpl.apply(this, arguments);
2459
550
  }
2460
551
 
2461
552
  async prefillWithLogits(prompt, options = {}) {
2462
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2463
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
2464
- throw new Error('Generation already in progress');
2465
- }
2466
- assertIncrementalDecodeSupport(this.#state, 'prefillWithLogits');
2467
- const totalStart = performance.now();
2468
- this._resetDecodeRuntimeState();
2469
- this.#state.stats.gpuTimePrefillMs = undefined;
2470
- this.#state.stats.prefillProfileSteps = [];
2471
- const opts = resolvePrefillOptions(this.#state, options);
2472
- const diagnosticsEnabled = options?.diagnostics?.enabled === true
2473
- || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
2474
- if (diagnosticsEnabled) {
2475
- const captureConfig = {
2476
- ...createDefaultCaptureConfig(),
2477
- enabled: true,
2478
- defaultLevel: CAPTURE_LEVELS.SLICE,
2479
- ...(options?.diagnostics?.captureConfig ?? {}),
2480
- };
2481
- validateCaptureConfig(captureConfig);
2482
- this.#state.operatorDiagnostics = {
2483
- enabled: true,
2484
- captureConfig,
2485
- emitter: new OperatorEventEmitter({
2486
- modelHash: this.#state.manifest?.modelId ?? null,
2487
- runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
2488
- executionPlanHash: opts.executionPlan?.id ?? null,
2489
- }),
2490
- tsirFixture: null,
2491
- };
2492
- }
2493
- try {
2494
- const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
2495
-
2496
- const snapshot = this.#state.kvCache?.clone();
2497
- if (!snapshot) {
2498
- throw new Error('KV cache unavailable after prefill');
2499
- }
2500
-
2501
- return {
2502
- cache: snapshot,
2503
- seqLen: this.#state.currentSeqLen,
2504
- tokens: inputIds,
2505
- logits,
2506
- phase: {
2507
- ...(prefillPhase ?? {}),
2508
- totalMs: performance.now() - totalStart,
2509
- },
2510
- linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
2511
- };
2512
- } finally {
2513
- this._closeFinitenessFallbackWindow(opts);
2514
- this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
2515
- ? {
2516
- enabled: true,
2517
- timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
2518
- recordCount: this.#state.operatorDiagnostics.emitter.length,
2519
- }
2520
- : null;
2521
- this.#state.operatorDiagnostics = null;
2522
- }
553
+ return prefillWithLogitsImpl.apply(this, arguments);
2523
554
  }
2524
555
 
2525
556
  async prefillWithTokenLogits(prompt, tokenIds, options = {}) {
2526
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2527
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
2528
- throw new Error('Generation already in progress');
2529
- }
2530
- assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogits');
2531
- const totalStart = performance.now();
2532
- const selectedTokenIds = normalizeSelectedLogitTokenIds(
2533
- tokenIds,
2534
- this.#state.modelConfig.vocabSize,
2535
- 'prefillWithTokenLogits.tokenIds'
2536
- );
2537
- this._resetDecodeRuntimeState();
2538
- this.#state.stats.gpuTimePrefillMs = undefined;
2539
- this.#state.stats.prefillProfileSteps = [];
2540
- const opts = resolvePrefillOptions(this.#state, options);
2541
- opts._selectedLogitTokenIds = selectedTokenIds;
2542
- try {
2543
- const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogits');
2544
- const logitsByTokenId = {};
2545
- for (let index = 0; index < selectedTokenIds.length; index += 1) {
2546
- logitsByTokenId[selectedTokenIds[index]] = logits[index];
2547
- }
2548
- return {
2549
- seqLen: this.#state.currentSeqLen,
2550
- tokens: inputIds,
2551
- tokenIds: selectedTokenIds,
2552
- logits,
2553
- logitsByTokenId,
2554
- phase: {
2555
- ...(prefillPhase ?? {}),
2556
- totalMs: performance.now() - totalStart,
2557
- selectedTokenCount: selectedTokenIds.length,
2558
- },
2559
- };
2560
- } finally {
2561
- this._closeFinitenessFallbackWindow(opts);
2562
- }
557
+ return prefillWithTokenLogitsImpl.apply(this, arguments);
2563
558
  }
2564
559
 
2565
560
  async prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
2566
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2567
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
2568
- throw new Error('Generation already in progress');
2569
- }
2570
- assertIncrementalDecodeSupport(this.#state, 'prefillWithTokenLogitsFromKV');
2571
- const totalStart = performance.now();
2572
- if (!prefix || typeof prefix !== 'object' || !prefix.cache) {
2573
- throw new Error('prefillWithTokenLogitsFromKV.prefix must be a KV cache snapshot.');
2574
- }
2575
- if (!Number.isInteger(prefix.seqLen) || prefix.seqLen < 0) {
2576
- throw new Error('prefillWithTokenLogitsFromKV.prefix.seqLen must be a non-negative integer.');
2577
- }
2578
- if (!Array.isArray(prefix.tokens)) {
2579
- throw new Error('prefillWithTokenLogitsFromKV.prefix.tokens must be an array.');
2580
- }
2581
- const selectedTokenIds = normalizeSelectedLogitTokenIds(
2582
- tokenIds,
2583
- this.#state.modelConfig.vocabSize,
2584
- 'prefillWithTokenLogitsFromKV.tokenIds'
2585
- );
2586
-
2587
- this.#state.kvCache = prefix.cache.clone();
2588
- if (this.#state.useGPU && this.#state.kvCache) {
2589
- const device = getDevice();
2590
- if (device) {
2591
- this.#state.kvCache.setGPUContext({ device });
2592
- }
2593
- }
2594
- if (
2595
- hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
2596
- && prefix.linearAttention == null
2597
- ) {
2598
- throw new Error(
2599
- 'Prefix snapshot is missing linear_attention recurrent state. ' +
2600
- 'Regenerate the prefix snapshot using the current runtime.'
2601
- );
2602
- }
2603
- this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
2604
- this.#state.linearAttentionRuntime,
2605
- prefix.linearAttention ?? null
2606
- );
2607
- this.#state.currentSeqLen = prefix.seqLen;
2608
- this.#state.stats.gpuTimePrefillMs = undefined;
2609
- this.#state.stats.prefillProfileSteps = [];
2610
- this.#state.decodeRing?.reset();
2611
-
2612
- const opts = resolvePrefillOptions(this.#state, options);
2613
- opts._selectedLogitTokenIds = selectedTokenIds;
2614
- try {
2615
- const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithTokenLogitsFromKV');
2616
- const logitsByTokenId = {};
2617
- for (let index = 0; index < selectedTokenIds.length; index += 1) {
2618
- logitsByTokenId[selectedTokenIds[index]] = logits[index];
2619
- }
2620
- return {
2621
- seqLen: this.#state.currentSeqLen,
2622
- prefixTokens: prefix.tokens,
2623
- tokens: inputIds,
2624
- tokenIds: selectedTokenIds,
2625
- logits,
2626
- logitsByTokenId,
2627
- phase: {
2628
- ...(prefillPhase ?? {}),
2629
- totalMs: performance.now() - totalStart,
2630
- selectedTokenCount: selectedTokenIds.length,
2631
- prefixTokens: prefix.tokens.length,
2632
- },
2633
- };
2634
- } finally {
2635
- this._closeFinitenessFallbackWindow(opts);
2636
- }
561
+ return prefillWithTokenLogitsFromKVImpl.apply(this, arguments);
2637
562
  }
2638
563
 
2639
-
2640
564
  async *generateWithPrefixKV(prefix, prompt, options = {}) {
2641
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
2642
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
2643
- assertIncrementalDecodeSupport(this.#state, 'generateWithPrefixKV');
2644
-
2645
- validateCallTimeOptions(options);
2646
-
2647
- // Apply snapshot
2648
- this.#state.kvCache = prefix.cache.clone();
2649
- if (this.#state.useGPU && this.#state.kvCache) {
2650
- const device = getDevice();
2651
- if (device) {
2652
- this.#state.kvCache.setGPUContext({ device });
2653
- }
2654
- }
2655
- if (
2656
- hasLinearAttentionLayers(this.#state.modelConfig.layerTypes)
2657
- && prefix.linearAttention == null
2658
- ) {
2659
- throw new Error(
2660
- 'Prefix snapshot is missing linear_attention recurrent state. ' +
2661
- 'Regenerate the prefix snapshot using the current runtime.'
2662
- );
2663
- }
2664
- this.#state.linearAttentionRuntime = restoreLinearAttentionRuntime(
2665
- this.#state.linearAttentionRuntime,
2666
- prefix.linearAttention ?? null
2667
- );
2668
- this.#state.currentSeqLen = prefix.seqLen;
2669
-
2670
- this.#state.isGenerating = true;
2671
- this.#state.decodeStepCount = 0;
2672
- resetActiveExecutionPlan(this.#state);
2673
- this.#state.stats.gpuTimePrefillMs = undefined;
2674
- this.#state.stats.gpuTimeDecodeMs = undefined;
2675
- this.#state.stats.prefillProfileSteps = [];
2676
- this.#state.decodeRing?.reset();
2677
- this.#state.stats.decodeRecordMs = 0;
2678
- this.#state.stats.decodeRecordOps = 0;
2679
- this.#state.stats.decodeRecordPasses = 0;
2680
- this.#state.stats.decodeRecordOpLabels = {};
2681
- this.#state.stats.decodeSubmitWaitMs = 0;
2682
- this.#state.stats.decodeReadbackWaitMs = 0;
2683
- this.#state.stats.decodeReadbackMapWaitMs = 0;
2684
- this.#state.stats.decodeReadbackCleanupMs = 0;
2685
- this.#state.stats.decodeReadbackCopyMs = 0;
2686
- this.#state.stats.prefillRecordMs = 0;
2687
- this.#state.stats.prefillRecordOps = 0;
2688
- this.#state.stats.prefillRecordPasses = 0;
2689
- this.#state.stats.prefillRecordOpLabels = {};
2690
- this.#state.stats.prefillSubmitWaitMs = 0;
2691
- this.#state.stats.ttftMs = 0;
2692
- const startTime = performance.now();
2693
-
2694
- const opts = resolveGenerateOptions(this.#state, options);
2695
-
2696
- try {
2697
- const processedPrompt = resolvePromptInput(this.#state, prompt, opts.useChatTemplate, 'generateWithPrefixKV');
2698
-
2699
- const inputIds = this.#state.tokenizer.encode(processedPrompt);
2700
- this._assertTokenIdsInRange(inputIds, 'generateWithPrefixKV.encode');
2701
- const generatedIds = [...prefix.tokens, ...inputIds];
2702
- const promptTokenCount = generatedIds.length;
2703
- this.#state.stats.prefillTokens = inputIds.length;
2704
-
2705
- const prefillStart = performance.now();
2706
- const prefillLogits = await this._prefill(inputIds, opts);
2707
- this.#state.stats.prefillTimeMs = performance.now() - prefillStart;
2708
-
2709
- applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
2710
- const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
2711
- const firstToken = sample(prefillLogits, {
2712
- temperature: opts.temperature,
2713
- topP: opts.topP,
2714
- topK: opts.topK,
2715
- padTokenId,
2716
- seed: opts.seed,
2717
- suppressTokenIds: opts.suppressTokenIds,
2718
- });
2719
-
2720
- const decodeRuntime = {
2721
- stopTokenIds: this.#state.modelConfig.stopTokenIds,
2722
- eosToken: this.#state.tokenizer.getSpecialTokens?.()?.eos,
2723
- stopSequenceStart: promptTokenCount,
2724
- decodeToken: (tokenId) => this.#state.tokenizer.decode([tokenId], true, false),
2725
- logBatchPath: false,
2726
- };
2727
- generatedIds.push(firstToken);
2728
- this.#state.stats.ttftMs = performance.now() - startTime;
2729
- const firstText = resolveTokenText(
2730
- this.#state.tokenizer,
2731
- [firstToken],
2732
- `[${firstToken}]`,
2733
- (tokens) => this.#state.tokenizer?.decode?.(tokens, true, false),
2734
- (tokens) => this.#state.tokenizer?.decode?.(tokens, false, false)
2735
- );
2736
- yield firstText;
2737
- if (options.onToken) options.onToken(firstToken, firstText);
2738
-
2739
- if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
2740
- this._recordStopReason('stop-token-or-sequence', firstToken);
2741
- this.#state.stats.decodeTimeMs = 0;
2742
- this.#state.stats.tokensGenerated = 1;
2743
- this.#state.stats.decodeTokens = 1;
2744
- } else {
2745
- yield* this._runDecodeLoop(generatedIds, opts, options, decodeRuntime);
2746
- }
2747
- this.#state.stats.totalTimeMs = performance.now() - startTime;
2748
- } finally {
2749
- this._closeFinitenessFallbackWindow(opts);
2750
- resetActiveExecutionPlan(this.#state);
2751
- this.#state.isGenerating = false;
2752
- }
565
+ return yield* generateWithPrefixKVImpl.apply(this, arguments);
2753
566
  }
2754
567
 
2755
568
  // ==========================================================================
@@ -2757,1221 +570,34 @@ export class PipelineGenerator {
2757
570
  // ==========================================================================
2758
571
 
2759
572
  async *_runDecodeLoop(generatedIds, opts, options, runtime) {
2760
- const {
2761
- stopTokenIds,
2762
- eosToken,
2763
- stopSequenceStart,
2764
- decodeToken,
2765
- logBatchPath = false,
2766
- emitMode = 'text',
2767
- } = runtime;
2768
-
2769
- let tokensGenerated = 1;
2770
- markKernelCacheWarmed();
2771
-
2772
- // Step 4: Lazily initialise PLE buffer cache for decode-path slice reuse.
2773
- const pleHiddenSize = Number(this.#state.modelConfig.hiddenSizePerLayerInput ?? 0);
2774
- if (pleHiddenSize > 0 && !this.#state.pleCache) {
2775
- const activationDtype = resolveActiveExecutionPlan(this.#state).activationDtype;
2776
- const bpe = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
2777
- this.#state.pleCache = createPleBufferCache(
2778
- this.#state.modelConfig.numLayers,
2779
- pleHiddenSize * bpe,
2780
- );
2781
- }
2782
- if (pleHiddenSize > 0) {
2783
- await primePleDecodeRuntimeCache(this.#state, generatedIds);
2784
- }
2785
-
2786
- const decodeStart = performance.now();
2787
- const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
2788
- this.#state.modelConfig.perLayerInputsSession ?? null,
2789
- this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
2790
- );
2791
- const lmHead = this.#state.weights.get('lm_head');
2792
- const embedBuffer = this.#state.weights.get('embed');
2793
- const hasCpuWeights = isCpuWeightBuffer(lmHead)
2794
- || isCpuWeightBuffer(embedBuffer)
2795
- || lmHead instanceof Float32Array
2796
- || embedBuffer instanceof Float32Array;
2797
- const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
2798
- const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
2799
- const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
2800
- const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
2801
- const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
2802
- config: this.#state.modelConfig,
2803
- weights: this.#state.weights,
2804
- });
2805
- const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
2806
- config: this.#state.modelConfig,
2807
- weights: this.#state.weights,
2808
- perLayerInputsSession: resolvedPerLayerInputsSession,
2809
- });
2810
- const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
2811
- const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
2812
- hasRangeBackedPerLayerInputs,
2813
- pleHotVocabularyRuntime,
2814
- tokenId: generatedIds[generatedIds.length - 1] ?? null,
2815
- });
2816
- const initialHotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
2817
- const executionPlan = opts.executionPlan;
2818
- let useBatchPath = replayPrefillDecode
2819
- ? false
2820
- : shouldUseBatchDecode({
2821
- batchSize: executionPlan.batchSize,
2822
- useGPU: this.#state.useGPU,
2823
- gpuSamplingAvailable,
2824
- disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
2825
- disableCommandBatching: executionPlan.disableCommandBatching,
2826
- isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
2827
- finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
2828
- hasLinearAttentionLayers: hasLinearLayers,
2829
- selfSpeculationEnabled: opts.speculation?.mode === 'self' && !initialHotVocabularyBatchDecodeAvailable,
2830
- hasRangeBackedPerLayerInputs,
2831
- });
2832
- if (!useBatchPath) {
2833
- let reason = null;
2834
- if (replayPrefillDecode) reason = 'replay_prefill_decode';
2835
- else if (hasCpuWeights) reason = 'cpu_weights';
2836
- else if (!this.#state.useGPU) reason = 'no_gpu';
2837
- else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
2838
- else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
2839
- else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
2840
- else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
2841
- else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
2842
- else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
2843
- else if (this._hasFinitenessFallbackWindow()) reason = 'finiteness_fallback_window';
2844
- else if (hasLinearLayers) reason = 'linear_attention_layers';
2845
- this.#state.stats.decodeMode = replayPrefillDecode
2846
- ? 'replay_prefill'
2847
- : (opts.speculation?.mode === 'self' ? 'self_speculation' : 'single_token');
2848
- this.#state.stats.batchGuardReason = reason;
2849
- } else {
2850
- this.#state.stats.decodeMode = hasRangeBackedPerLayerInputs
2851
- ? 'batched_gpu_stepwise_ple'
2852
- : 'batched_gpu';
2853
- this.#state.stats.batchGuardReason = null;
2854
- }
2855
-
2856
- const readbackInterval = executionPlan.readbackInterval;
2857
- const intervalBatches = readbackInterval == null ? 1 : readbackInterval;
2858
- const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
2859
-
2860
- const decodeSingleTokenViaLogits = async () => this._decodeNextTokenViaLogits(generatedIds, opts);
2861
-
2862
- if (logBatchPath && useBatchPath) {
2863
- log.debug(
2864
- 'Pipeline',
2865
- `Using batch decode path with batchSize=${executionPlan.batchSize}, stopCheckMode=${executionPlan.stopCheckMode}, readbackInterval=${readbackInterval}`
2866
- );
2867
- }
2868
-
2869
- while (tokensGenerated < opts.maxTokens) {
2870
- if (options.signal?.aborted) {
2871
- this._recordStopReason('aborted');
2872
- break;
2873
- }
2874
- if (this._hasFinitenessFallbackWindow() && useBatchPath) {
2875
- useBatchPath = false;
2876
- }
2877
- const hotVocabularyBatchDecodeAvailable = resolveCurrentHotVocabularyBatchDecodeAvailable();
2878
-
2879
- if (useBatchPath) {
2880
- const remaining = opts.maxTokens - tokensGenerated;
2881
- const maxBatchDecodeTokens = resolveMaxBatchDecodeTokens({
2882
- hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
2883
- hasGpuSplitPerLayerInputs,
2884
- hasLinearAttentionLayers: hasLinearLayers,
2885
- modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
2886
- activationDtype: executionPlan.activationDtype,
2887
- currentSeqLen: this.#state.currentSeqLen,
2888
- maxDecodeTokens: opts.maxTokens,
2889
- numLayers: this.#state.modelConfig.numLayers,
2890
- hiddenSize: this.#state.modelConfig.hiddenSize,
2891
- configuredMaxBatchDecodeTokens: executionPlan.maxBatchDecodeTokens,
2892
- });
2893
- const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
2894
- const boundedBatchTokens = maxBatchDecodeTokens == null
2895
- ? requestedBatchTokens
2896
- : Math.min(requestedBatchTokens, maxBatchDecodeTokens);
2897
- const thisBatchSize = Math.min(boundedBatchTokens, remaining);
2898
- this.#state.batchingStats.requestedBatchTokens = Math.max(
2899
- this.#state.batchingStats.requestedBatchTokens ?? 0,
2900
- requestedBatchTokens
2901
- );
2902
- this.#state.batchingStats.effectiveBatchTokens = Math.max(
2903
- this.#state.batchingStats.effectiveBatchTokens ?? 0,
2904
- thisBatchSize
2905
- );
2906
- if (maxBatchDecodeTokens != null) {
2907
- this.#state.batchingStats.maxBatchTokenCap = Math.max(
2908
- this.#state.batchingStats.maxBatchTokenCap ?? 0,
2909
- maxBatchDecodeTokens
2910
- );
2911
- }
2912
- if (thisBatchSize < requestedBatchTokens) {
2913
- this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
2914
- }
2915
- const lastToken = generatedIds[generatedIds.length - 1];
2916
- const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
2917
- ? {
2918
- ...executionPlan,
2919
- batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
2920
- readbackInterval: readbackInterval == null
2921
- ? null
2922
- : Math.min(intervalBatches, thisBatchSize),
2923
- }
2924
- : executionPlan;
2925
- const batchOpts = boundedExecutionPlan === executionPlan
2926
- ? opts
2927
- : { ...opts, executionPlan: boundedExecutionPlan };
2928
-
2929
- try {
2930
- const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
2931
- let batchTokens = [];
2932
- let hitStop = false;
2933
- let stopTokenId = null;
2934
- for (const tokenId of batchResult.tokens) {
2935
- if (isStopToken(tokenId, stopTokenIds, eosToken)) {
2936
- hitStop = true;
2937
- stopTokenId = tokenId;
2938
- break;
2939
- }
2940
- generatedIds.push(tokenId);
2941
- tokensGenerated++;
2942
- if (emitMode === 'token') {
2943
- yield tokenId;
2944
- if (options.onToken) options.onToken(tokenId, '');
2945
- batchTokens.push({ id: tokenId, text: '' });
2946
- } else {
2947
- const tokenText = decodeToken(tokenId);
2948
- yield tokenText;
2949
- if (options.onToken) options.onToken(tokenId, tokenText);
2950
- batchTokens.push({ id: tokenId, text: tokenText });
2951
- }
2952
- if (batchTokens.length === executionPlan.batchSize) {
2953
- if (options.onBatch) options.onBatch(batchTokens);
2954
- batchTokens = [];
2955
- }
2956
- }
2957
- if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
2958
- if (opts.stopSequences.length > 0) {
2959
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2960
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2961
- this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
2962
- break;
2963
- }
2964
- }
2965
- if (hitStop) {
2966
- this._recordStopReason('stop-token', stopTokenId);
2967
- break;
2968
- }
2969
- if (shouldDisableBatchDecodeAfterShortBatch({
2970
- hitStop,
2971
- actualCount: batchResult.actualCount,
2972
- requestedCount: thisBatchSize,
2973
- })) {
2974
- useBatchPath = false;
2975
- continue;
2976
- }
2977
- } catch (error) {
2978
- log.warn('Pipeline', `Batch decode failed, falling back to single-token: ${error}`);
2979
- useBatchPath = false;
2980
- let nextToken;
2981
- try {
2982
- nextToken = await decodeSingleTokenViaLogits();
2983
- } catch (singleTokenError) {
2984
- if (this._shouldUseFinitenessFallback(singleTokenError, `decode-batch-step-${tokensGenerated}`)) {
2985
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at batch step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
2986
- nextToken = await this._retryDecodeStepWithFinitenessWindow(
2987
- generatedIds,
2988
- opts,
2989
- `decode-batch-step-${tokensGenerated}`
2990
- );
2991
- } else {
2992
- throw singleTokenError;
2993
- }
2994
- }
2995
- generatedIds.push(nextToken);
2996
- tokensGenerated++;
2997
- if (emitMode === 'token') {
2998
- yield nextToken;
2999
- if (options.onToken) options.onToken(nextToken, '');
3000
- } else {
3001
- const tokenText = decodeToken(nextToken);
3002
- yield tokenText;
3003
- if (options.onToken) options.onToken(nextToken, tokenText);
3004
- }
3005
- this._consumeFinitenessFallbackToken(opts);
3006
- if (isStopToken(nextToken, stopTokenIds, eosToken)) {
3007
- this._recordStopReason('stop-token', nextToken);
3008
- break;
3009
- }
3010
- }
3011
- } else if (opts.speculation?.mode === 'self') {
3012
- // Self-speculation: decode one base token plus a configurable burst of
3013
- // speculative tokens per iteration. Same-model speculation always
3014
- // accepts under greedy because the model is deterministic — both base
3015
- // and speculative use the same weights and state. The benefit is
3016
- // amortizing per-iteration overhead for models where batch decode is
3017
- // disabled (e.g., linear attention).
3018
- const speculativeBurstTokens = opts.speculation.tokens;
3019
- if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
3020
- throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
3021
- }
3022
- const doSpecDecode = hasLinearLayers
3023
- ? () => this._decodeStep(generatedIds, opts)
3024
- : decodeSingleTokenViaLogits;
3025
-
3026
- // Base decode
3027
- let baseToken;
3028
- try {
3029
- baseToken = await doSpecDecode();
3030
- } catch (error) {
3031
- if (this._shouldUseFinitenessFallback(error, `spec-base-${tokensGenerated}`)) {
3032
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:base). Retrying.`);
3033
- baseToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-base-${tokensGenerated}`);
3034
- } else {
3035
- throw error;
3036
- }
3037
- }
3038
- generatedIds.push(baseToken);
3039
- tokensGenerated++;
3040
- if (emitMode === 'token') {
3041
- yield baseToken;
3042
- if (options.onToken) options.onToken(baseToken, '');
3043
- } else {
3044
- const text = decodeToken(baseToken);
3045
- yield text;
3046
- if (options.onToken) options.onToken(baseToken, text);
3047
- }
3048
- this._consumeFinitenessFallbackToken(opts);
3049
-
3050
- if (isStopToken(baseToken, stopTokenIds, eosToken)) {
3051
- this._recordStopReason('stop-token', baseToken);
3052
- break;
3053
- }
3054
- if (tokensGenerated >= opts.maxTokens) break;
3055
- if (opts.stopSequences.length > 0) {
3056
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
3057
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
3058
- this._recordStopReason('stop-sequence', baseToken);
3059
- break;
3060
- }
3061
- }
3062
-
3063
- for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
3064
- if (tokensGenerated >= opts.maxTokens) {
3065
- break;
3066
- }
3067
- let specToken;
3068
- try {
3069
- specToken = await doSpecDecode();
3070
- } catch (error) {
3071
- if (this._shouldUseFinitenessFallback(error, `spec-extra-${tokensGenerated}`)) {
3072
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated} (speculation:spec). Retrying.`);
3073
- specToken = await this._retryDecodeStepWithFinitenessWindow(generatedIds, opts, `spec-extra-${tokensGenerated}`);
3074
- } else {
3075
- throw error;
3076
- }
3077
- }
3078
- generatedIds.push(specToken);
3079
- tokensGenerated++;
3080
- this.#state.stats.speculationAttempts = (this.#state.stats.speculationAttempts ?? 0) + 1;
3081
- this.#state.stats.speculationAccepted = (this.#state.stats.speculationAccepted ?? 0) + 1;
3082
- if (emitMode === 'token') {
3083
- yield specToken;
3084
- if (options.onToken) options.onToken(specToken, '');
3085
- } else {
3086
- const text = decodeToken(specToken);
3087
- yield text;
3088
- if (options.onToken) options.onToken(specToken, text);
3089
- }
3090
- this._consumeFinitenessFallbackToken(opts);
3091
-
3092
- if (opts.debug || opts.benchmark) {
3093
- const elapsedMs = performance.now() - decodeStart;
3094
- const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
3095
- log.debug('Decode', `#${tokensGenerated} speculation:self (${tokPerSec.toFixed(2)} tok/s avg)`);
3096
- }
3097
-
3098
- if (isStopToken(specToken, stopTokenIds, eosToken)) {
3099
- this._recordStopReason('stop-token', specToken);
3100
- break;
3101
- }
3102
- if (opts.stopSequences.length > 0) {
3103
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
3104
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
3105
- this._recordStopReason('stop-sequence', specToken);
3106
- break;
3107
- }
3108
- }
3109
- }
3110
- if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
3111
- this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
3112
- break;
3113
- }
3114
- if (opts.stopSequences.length > 0) {
3115
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
3116
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
3117
- this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
3118
- break;
3119
- }
3120
- }
3121
- } else {
3122
- const tokenStart = performance.now();
3123
- let nextToken;
3124
- try {
3125
- nextToken = hasLinearLayers
3126
- ? await this._decodeStep(generatedIds, opts)
3127
- : await decodeSingleTokenViaLogits();
3128
- } catch (error) {
3129
- if (this._shouldUseFinitenessFallback(error, `decode-step-${tokensGenerated}`)) {
3130
- log.warn('Pipeline', `FinitenessGuard caught NaN/Inf at step ${tokensGenerated}. Truncating KV cache and retrying token with F32 precision.`);
3131
- nextToken = await this._retryDecodeStepWithFinitenessWindow(
3132
- generatedIds,
3133
- opts,
3134
- `decode-step-${tokensGenerated}`
3135
- );
3136
- } else {
3137
- throw error;
3138
- }
3139
- }
3140
- const tokenTime = performance.now() - tokenStart;
3141
- generatedIds.push(nextToken);
3142
- tokensGenerated++;
3143
-
3144
- // Step 5: Fire-and-forget prefetch of next token's PLE row.
3145
- if (pleHiddenSize > 0) {
3146
- const pleWeights = this.#state.weights.get('per_layer_inputs');
3147
- if (pleWeights?.embedTokensPerLayer) {
3148
- this.#state.plePrefetchPending = prefetchPerLayerRow(
3149
- nextToken,
3150
- pleWeights.embedTokensPerLayer,
3151
- this.#state.modelConfig.numLayers * pleHiddenSize,
3152
- resolvedPerLayerInputsSession,
3153
- );
3154
- }
3155
- }
3156
-
3157
- const tokenText = emitMode === 'token' ? '' : decodeToken(nextToken);
3158
- if (emitMode === 'token') {
3159
- yield nextToken;
3160
- if (options.onToken) options.onToken(nextToken, '');
3161
- } else {
3162
- yield tokenText;
3163
- if (options.onToken) options.onToken(nextToken, tokenText);
3164
- }
3165
- this._consumeFinitenessFallbackToken(opts);
3166
-
3167
- if (opts.debug || opts.benchmark) {
3168
- const elapsedMs = performance.now() - decodeStart;
3169
- const tokPerSec = (tokensGenerated / elapsedMs) * 1000;
3170
- log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
3171
- }
3172
-
3173
- if (isStopToken(nextToken, stopTokenIds, eosToken)) {
3174
- this._recordStopReason('stop-token', nextToken);
3175
- break;
3176
- }
3177
- if (opts.stopSequences.length > 0) {
3178
- const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
3179
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
3180
- this._recordStopReason('stop-sequence', nextToken);
3181
- break;
3182
- }
3183
- }
3184
- }
3185
- }
3186
-
3187
- if (!this.#state.stats.stopReason) {
3188
- this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
3189
- }
3190
- this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
3191
- this.#state.stats.tokensGenerated = tokensGenerated;
3192
- this.#state.stats.decodeTokens = tokensGenerated;
3193
- this.#state.stats.batching = { ...this.#state.batchingStats };
573
+ return yield* _runDecodeLoopImpl.apply(this, arguments);
3194
574
  }
3195
575
 
3196
576
  async _prefillToHidden(inputIds, opts) {
3197
- // Internal-only: reuse the main prefill implementation but stop before logits.
3198
- return this._prefill(inputIds, { ...opts, _returnHidden: true });
577
+ return _prefillToHiddenImpl.apply(this, arguments);
3199
578
  }
3200
579
 
3201
-
3202
580
  async _prefill(inputIds, opts) {
3203
- const numTokens = inputIds.length;
3204
- const config = this.#state.modelConfig;
3205
- const selectedLogitTokenIds = normalizeSelectedLogitTokenIds(
3206
- opts?._selectedLogitTokenIds,
3207
- config.vocabSize,
3208
- '_prefill.selectedLogitTokenIds'
3209
- );
3210
- const startPos = this.#state.currentSeqLen;
3211
- const tracePrefillEnabled = isTraceEnabled('perf');
3212
- const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
3213
- const returnHidden = opts?._returnHidden === true;
3214
- const embeddingInputIds = resolvePrefillEmbeddingInputIds(
3215
- inputIds,
3216
- opts?.embeddingInputSpan ?? null,
3217
- '_prefill'
3218
- );
3219
- const multimodalBidirectionalSpan = resolvePrefillMultimodalBidirectionalSpan(
3220
- inputIds,
3221
- opts?.multimodalBidirectionalSpan ?? null,
3222
- '_prefill'
3223
- );
3224
- if (embeddingInputIds !== inputIds) {
3225
- this._assertTokenIdsInRange(embeddingInputIds, '_prefill.embeddingInputIds');
3226
- }
3227
- const embeddingOverride = normalizePrefixEmbeddingOverride(
3228
- opts?.embeddingOverrides ?? null,
3229
- config.hiddenSize,
3230
- numTokens,
3231
- '_prefill'
3232
- );
3233
- this.#state.stats.gpuTimePrefillMs = undefined;
3234
- this.#state.stats.prefillProfileSteps = [];
3235
-
3236
- if (startPos === 0 && hasLinearAttentionLayers(config.layerTypes)) {
3237
- this.#state.linearAttentionRuntime = resetLinearAttentionRuntime(this.#state.linearAttentionRuntime);
3238
- }
3239
- if (startPos === 0) {
3240
- for (const [, convState] of this.#state.convLayerStates) {
3241
- if (convState.convStateGPU && convState.hiddenSize && convState.kernelSize) {
3242
- uploadData(convState.convStateGPU, new Float32Array(convState.hiddenSize * (convState.kernelSize - 1)));
3243
- }
3244
- }
3245
- }
3246
-
3247
- const embedBufferRaw = this.#state.weights.get('embed');
3248
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
3249
- throw new Error('Embed buffer not found or not a supported buffer type');
3250
- }
3251
- const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
3252
- const embedDtype = isCpuWeightBuffer(embedBufferRaw)
3253
- ? embedBufferRaw.dtype
3254
- : getWeightDtype(embedBufferRaw);
3255
- const embedMetadata = getWeightMetadata(embedBufferRaw);
3256
- if (opts.debug) {
3257
- const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
3258
- log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
3259
- }
3260
-
3261
- const device = getDevice();
3262
- const useCheckpoints = opts.debugLayers && opts.debugLayers.length > 0;
3263
- const disableCommandBatching = shouldDisablePrefillCommandBatching(
3264
- this.#state,
3265
- opts,
3266
- multimodalBidirectionalSpan
3267
- );
3268
- const createRecorder = (label) => {
3269
- if (!device || disableCommandBatching) return undefined;
3270
- return opts.profile
3271
- ? createProfilingRecorder(label, device)
3272
- : createCommandRecorder(label, { recordLabels: opts.debug === true || opts.benchmark === true }, device);
3273
- };
3274
- const recorder = createRecorder('prefill');
3275
- const debugCheckBuffer = this.#state.debug
3276
- ? (buffer, label, numTokens, expectedDim) =>
3277
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
3278
- : undefined;
3279
- const context = buildLayerContext(
3280
- this.#state,
3281
- recorder,
3282
- false,
3283
- opts.debugLayers,
3284
- debugCheckBuffer,
3285
- opts.executionPlan
3286
- );
3287
- context.trainingCapture = opts?._trainingCapture ?? null;
3288
- context.currentTokenIds = inputIds;
3289
- context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
3290
- context.skipKVCacheWrites = returnHidden
3291
- && opts?._embeddingOnly === true
3292
- && this.#state.runtimeConfig?.inference?.session?.skipEmbeddingKVCacheWrites === true;
3293
- context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
3294
- ? null
3295
- : {
3296
- start: startPos + multimodalBidirectionalSpan.offset,
3297
- length: multimodalBidirectionalSpan.length,
3298
- };
3299
- let gpuTimePrefillMs = 0;
3300
- let hasGpuTimePrefill = false;
3301
- const recordProfile = async (rec) => {
3302
- if (!opts.profile || !rec?.isProfilingEnabled()) return;
3303
- const timings = await rec.resolveProfileTimings();
3304
- const total = sumProfileTimings(timings);
3305
- if (total !== null) {
3306
- gpuTimePrefillMs += total;
3307
- hasGpuTimePrefill = true;
3308
- }
3309
- if (timings) {
3310
- recordPrefillProfileStep(this.#state, {
3311
- label: rec.label,
3312
- timings,
3313
- totalMs: total ?? undefined,
3314
- });
3315
- log.warn('Profile', `Prefill (${rec.label}):`);
3316
- log.warn('Profile', CommandRecorder.formatProfileReport(timings));
3317
- }
3318
- };
3319
-
3320
- const benchmarkSubmits = opts.debug;
3321
- if (benchmarkSubmits) {
3322
- setTrackSubmits(true);
3323
- resetSubmitStats();
3324
- }
3325
-
3326
- const preserveBufferAcrossRecorderSubmit = (buffer, activeRecorder, label) => {
3327
- if (!activeRecorder || !isGpuBufferInstance(buffer)) {
3328
- return buffer;
3329
- }
3330
- const carryBuffer = acquireBuffer(
3331
- buffer.size,
3332
- typeof buffer.usage === 'number' ? buffer.usage : undefined,
3333
- label
3334
- );
3335
- activeRecorder.getEncoder().copyBufferToBuffer(buffer, 0, carryBuffer, 0, buffer.size);
3336
- activeRecorder.trackTemporaryBuffer(buffer);
3337
- return carryBuffer;
3338
- };
3339
-
3340
- const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
3341
- const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
3342
- if (tracePrefillEnabled) {
3343
- trace.perf('Prefill phase start', {
3344
- numTokens,
3345
- startPos,
3346
- numLayers: config.numLayers,
3347
- activationDtype,
3348
- returnHidden,
3349
- });
3350
- }
3351
- const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
3352
- let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
3353
- hiddenSize: config.hiddenSize,
3354
- vocabSize: config.vocabSize,
3355
- scaleEmbeddings: config.scaleEmbeddings,
3356
- embeddingScale: config.embeddingScale,
3357
- debug: opts.debug,
3358
- recorder,
3359
- transpose: this.#state.embeddingTranspose,
3360
- debugProbes: this.#state.runtimeConfig.shared.debug.probes,
3361
- operatorDiagnostics: this.#state.operatorDiagnostics,
3362
- activationDtype,
3363
- embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
3364
- embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
3365
- executionPolicies: this.#state.executionV1State?.policies ?? null,
3366
- });
3367
- if (tracePrefillEnabled) {
3368
- trace.perf('Prefill embed complete', {
3369
- numTokens,
3370
- elapsedMs: performance.now() - embedTraceStart,
3371
- });
3372
- }
3373
- let hiddenStates = baseEmbeddings;
3374
- let perLayerInputs = null;
3375
- const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
3376
- try {
3377
- hiddenStates = await applyPrefixEmbeddingOverride(
3378
- baseEmbeddings,
3379
- embeddingOverride,
3380
- config.hiddenSize,
3381
- '_prefill',
3382
- {
3383
- executionPolicies: this.#state.executionV1State?.policies ?? null,
3384
- transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
3385
- }
3386
- );
3387
- perLayerInputs = await preparePerLayerInputs(
3388
- embeddingInputIds,
3389
- embeddingInputIds === inputIds ? hiddenStates : baseEmbeddings,
3390
- context,
3391
- {
3392
- numTokens,
3393
- pleCache: this.#state.pleCache ?? null,
3394
- }
3395
- );
3396
- if (tracePrefillEnabled) {
3397
- trace.perf('Prefill per-layer inputs complete', {
3398
- numTokens,
3399
- elapsedMs: performance.now() - perLayerInputsTraceStart,
3400
- materialized: Array.isArray(perLayerInputs),
3401
- });
3402
- }
3403
- } catch (error) {
3404
- if (isGpuBufferInstance(hiddenStates?.buffer)) {
3405
- releaseBuffer(hiddenStates.buffer);
3406
- }
3407
- if (hiddenStates === baseEmbeddings) {
3408
- baseEmbeddings = null;
3409
- }
3410
- hiddenStates = null;
3411
- throw error;
3412
- } finally {
3413
- if (hiddenStates !== baseEmbeddings && isGpuBufferInstance(baseEmbeddings?.buffer)) {
3414
- releaseBuffer(baseEmbeddings.buffer);
3415
- }
3416
- baseEmbeddings = null;
3417
- }
3418
-
3419
- if (opts.debug && isGpuBufferInstance(hiddenStates)) {
3420
- if (recorder) {
3421
- hiddenStates = createTensor(
3422
- preserveBufferAcrossRecorderSubmit(hiddenStates.buffer, recorder, 'prefill_embed_carry'),
3423
- hiddenStates.dtype,
3424
- hiddenStates.shape,
3425
- hiddenStates.label
3426
- );
3427
- recordPrefillRecorderStats(this.#state, recorder);
3428
- await recorder.submitAndWait();
3429
- await recordProfile(recorder);
3430
- }
3431
- const debugReadbackSize = this.#state.runtimeConfig.shared.debug.pipeline.readbackSampleSize;
3432
- const sample = await readBuffer(hiddenStates, Math.min(debugReadbackSize, hiddenStates.size));
3433
- const f32 = decodeReadback(sample, activationDtype);
3434
- const nanCount = f32.filter(x => !Number.isFinite(x)).length;
3435
- let maxAbs = 0;
3436
- for (let i = 0; i < f32.length; i++) {
3437
- const abs = Math.abs(f32[i]);
3438
- if (abs > maxAbs) maxAbs = abs;
3439
- }
3440
- const first8 = Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ');
3441
- log.debug('Pipeline', `After embed: buffer.label=${hiddenStates.label}, buffer.size=${hiddenStates.size}, maxAbs=${maxAbs.toFixed(4)}`);
3442
- log.debug('Pipeline', `After embed first8=[${first8}], nan=${nanCount}/${f32.length}`);
3443
- }
3444
-
3445
- if (opts.debug) {
3446
- log.debug('Pipeline', `LAYER_LOOP_START: numLayers=${config.numLayers}, useGPU=${context.useGPU}`);
3447
- }
3448
-
3449
- if (this.#state.finitenessBuffer) {
3450
- const device = getDevice();
3451
- if (device) {
3452
- device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
3453
- }
3454
- }
3455
-
3456
- let currentRecorder = recorder;
3457
-
3458
- // Chunked recorder submission: submit every N layers to release tracked intermediate
3459
- // buffers, preventing unbounded memory growth during large prefills. Critical for
3460
- // replay_prefill models where each decode step re-runs a prefill-style layer pass.
3461
- const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
3462
- configuredPrefillChunkLayers: this.#state.runtimeConfig?.inference?.session?.prefillChunkLayers,
3463
- hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
3464
- numTokens,
3465
- });
3466
-
3467
- let currentHiddenBuffer = hiddenStates.buffer;
3468
- let prefillRecordMs = 0;
3469
- let prefillSubmitWaitMs = 0;
3470
- const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
3471
- const traceLayerHealthEnabled = isTraceEnabled('logits');
3472
- try {
3473
- for (let l = 0; l < config.numLayers; l++) {
3474
- // Per-layer hard cancellation: when the caller's AbortSignal aborts,
3475
- // exit the prefill loop between layer dispatches rather than
3476
- // continuing to burn GPU on superseded work. Granularity is one
3477
- // layer (~5-50ms), which is the fastest cancel granularity WebGPU
3478
- // exposes today.
3479
- if (opts?.signal?.aborted) {
3480
- const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
3481
- const err = new Error(reason);
3482
- err.name = "AbortError";
3483
- err.code = "ABORT_ERR";
3484
- throw err;
3485
- }
3486
- context.recorder = currentRecorder;
3487
- context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
3488
-
3489
- const prevBuffer = currentHiddenBuffer;
3490
- const layerRecordStart = performance.now();
3491
- const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
3492
- prefillRecordMs += performance.now() - layerRecordStart;
3493
- if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
3494
- currentHiddenBuffer = layerOutput;
3495
- releasePerLayerInputBuffer(
3496
- context.perLayerInputBuffer,
3497
- currentRecorder,
3498
- context.decodeBuffers,
3499
- this.#state.pleCache ?? null
3500
- );
3501
- if (perLayerInputs) {
3502
- perLayerInputs[l] = null;
3503
- }
3504
- context.perLayerInputBuffer = null;
3505
-
3506
- const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
3507
- const isChunkBoundary = !isCheckpoint
3508
- && !traceLayerHealthEnabled
3509
- && currentRecorder
3510
- && l < config.numLayers - 1
3511
- && (l + 1) % prefillRecorderChunkLayers === 0;
3512
-
3513
- if (isCheckpoint && currentRecorder) {
3514
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3515
- currentHiddenBuffer,
3516
- currentRecorder,
3517
- 'prefill_checkpoint_carry'
3518
- );
3519
- recordPrefillRecorderStats(this.#state, currentRecorder);
3520
- await currentRecorder.submitAndWait();
3521
- await recordProfile(currentRecorder);
3522
- currentRecorder = undefined;
3523
- }
3524
-
3525
- if (traceLayerHealthEnabled && currentRecorder) {
3526
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3527
- currentHiddenBuffer,
3528
- currentRecorder,
3529
- 'prefill_trace_layer_health_carry'
3530
- );
3531
- const traceSubmitStart = performance.now();
3532
- recordPrefillRecorderStats(this.#state, currentRecorder);
3533
- await currentRecorder.submitAndWait();
3534
- await recordProfile(currentRecorder);
3535
- prefillSubmitWaitMs += performance.now() - traceSubmitStart;
3536
- await traceActivationHealth(
3537
- `PREFILL_LAYER_${l}_HEALTH`,
3538
- currentHiddenBuffer,
3539
- activationDtype,
3540
- numTokens * config.hiddenSize
3541
- );
3542
- currentRecorder = l < config.numLayers - 1
3543
- ? createRecorder('prefill-trace')
3544
- : undefined;
3545
- }
3546
-
3547
- const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
3548
- if (shouldDebug && !currentRecorder) {
3549
- const device = getDevice();
3550
- if (device) {
3551
- if (allowReadback(`pipeline.prefill.layer-${l}`)) {
3552
- try {
3553
- const sampleSize = config.hiddenSize * activationBytes;
3554
- const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
3555
- const readback = await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize);
3556
- const data = decodeReadback(readback, activationDtype);
3557
- let min = Infinity;
3558
- let max = -Infinity;
3559
- let maxAbs = 0;
3560
- for (const v of data) {
3561
- if (!Number.isFinite(v)) continue;
3562
- if (v < min) min = v;
3563
- if (v > max) max = v;
3564
- const av = Math.abs(v);
3565
- if (av > maxAbs) maxAbs = av;
3566
- }
3567
- const sample = Array.from(data).slice(0, 3).map(x => x.toFixed(3)).join(', ');
3568
- log.debug('Pipeline', `LAYER_${l}_LAST[pos=${numTokens - 1}]: min=${min.toFixed(3)}, max=${max.toFixed(3)}, maxAbs=${maxAbs.toFixed(2)}, sample=[${sample}]`);
3569
- } catch (e) {
3570
- log.debug('Pipeline', `LAYER_${l}_LAST: error reading buffer: ${e}`);
3571
- }
3572
- }
3573
- }
3574
- }
3575
-
3576
- if (isCheckpoint && useCheckpoints && l < config.numLayers - 1) {
3577
- currentRecorder = createRecorder('prefill-cont');
3578
- }
3579
-
3580
- if (prevBuffer !== currentHiddenBuffer) {
3581
- if (currentRecorder) {
3582
- currentRecorder.trackTemporaryBuffer(prevBuffer);
3583
- } else {
3584
- releaseBuffer(prevBuffer);
3585
- }
3586
- }
3587
-
3588
- if (isChunkBoundary) {
3589
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3590
- currentHiddenBuffer,
3591
- currentRecorder,
3592
- 'prefill_chunk_carry'
3593
- );
3594
- // Chunk boundary exists only to bound intermediate buffer lifetime.
3595
- // When the runtime opts into async chunk submit AND profile timings
3596
- // are not being collected, skip the CPU-GPU wait: queue order is
3597
- // preserved across submits and deferred cleanup still releases
3598
- // tracked buffers when GPU work completes. Profile runs keep the
3599
- // sync path because resolveProfileTimings requires the submitted
3600
- // work to be done.
3601
- const chunkSubmitMode = resolvePrefillChunkSubmitMode(
3602
- this.#state.runtimeConfig,
3603
- this.#state.modelConfig
3604
- );
3605
- const chunkSubmitStart = performance.now();
3606
- recordPrefillRecorderStats(this.#state, currentRecorder);
3607
- if (chunkSubmitMode === 'async' && !opts.profile) {
3608
- currentRecorder.submit({ cleanup: 'deferred' });
3609
- } else {
3610
- await currentRecorder.submitAndWait();
3611
- await recordProfile(currentRecorder);
3612
- }
3613
- prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
3614
- await traceActivationHealth(
3615
- `PREFILL_LAYER_${l}_HEALTH`,
3616
- currentHiddenBuffer,
3617
- activationDtype,
3618
- numTokens * config.hiddenSize
3619
- );
3620
- if (tracePrefillEnabled) {
3621
- trace.perf('Prefill chunk submitted', {
3622
- layer: l,
3623
- elapsedMs: performance.now() - layerLoopTraceStart,
3624
- prefillRecordMs,
3625
- prefillSubmitWaitMs,
3626
- });
3627
- }
3628
- currentRecorder = createRecorder('prefill-chunk');
3629
- }
3630
- }
3631
- if (tracePrefillEnabled) {
3632
- trace.perf('Prefill layer loop recorded', {
3633
- numTokens,
3634
- elapsedMs: performance.now() - layerLoopTraceStart,
3635
- prefillRecordMs,
3636
- prefillSubmitWaitMs,
3637
- });
3638
- }
3639
- } finally {
3640
- context.perLayerInputBuffer = null;
3641
- if (perLayerInputs) {
3642
- for (const buffer of perLayerInputs) {
3643
- releasePerLayerInputBuffer(
3644
- buffer,
3645
- currentRecorder,
3646
- context.decodeBuffers,
3647
- this.#state.pleCache ?? null
3648
- );
3649
- }
3650
- }
3651
- releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
3652
- this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
3653
- this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
3654
- }
3655
-
3656
- {
3657
- const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
3658
- if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
3659
- if (currentRecorder) {
3660
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3661
- currentHiddenBuffer,
3662
- currentRecorder,
3663
- 'prefill_tsir_drain_carry'
3664
- );
3665
- recordPrefillRecorderStats(this.#state, currentRecorder);
3666
- await currentRecorder.submitAndWait();
3667
- await recordProfile(currentRecorder);
3668
- currentRecorder = undefined;
3669
- }
3670
- await drainPendingTsirReads(tsirFixtureCfg);
3671
- }
3672
- }
3673
- if (this.#state.finitenessBuffer) {
3674
- if (currentRecorder) {
3675
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3676
- currentHiddenBuffer,
3677
- currentRecorder,
3678
- 'prefill_finiteness_carry'
3679
- );
3680
- recordPrefillRecorderStats(this.#state, currentRecorder);
3681
- await currentRecorder.submitAndWait();
3682
- await recordProfile(currentRecorder);
3683
- currentRecorder = undefined;
3684
- }
3685
- const isInfiniteData = await readBuffer(this.#state.finitenessBuffer, 16);
3686
- const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
3687
- const finitenessStatus = parseFinitenessStatusWords(u32, 0);
3688
- if (finitenessStatus.triggered) {
3689
- if (isGpuBufferInstance(currentHiddenBuffer)) {
3690
- releaseBuffer(currentHiddenBuffer);
3691
- }
3692
- throw new FinitenessError(`F16 bounds exceeded during prefill${finitenessStatus.metadata}`);
3693
- }
3694
- }
3695
-
3696
- if (benchmarkSubmits) {
3697
- logSubmitStats(`Prefill (${numTokens} tokens, ${config.numLayers} layers)`);
3698
- setTrackSubmits(false);
3699
- }
3700
-
3701
- if (opts.debug) {
3702
- log.debug('Pipeline', `LAYER_LOOP_DONE, currentHiddenBuffer type=${currentHiddenBuffer?.constructor?.name}`);
3703
- if (currentHiddenBuffer && allowReadback('pipeline.prefill.final-hidden')) {
3704
- const lastTokenOffset = (numTokens - 1) * config.hiddenSize * activationBytes;
3705
- const sampleSize = config.hiddenSize * activationBytes;
3706
- const data = decodeReadback(
3707
- await readBufferSlice(currentHiddenBuffer, lastTokenOffset, sampleSize),
3708
- activationDtype
3709
- );
3710
- const nanCount = Array.from(data).filter(x => !Number.isFinite(x)).length;
3711
- const nonZero = Array.from(data).filter(x => Number.isFinite(x) && x !== 0).slice(0, 5);
3712
- log.debug('Pipeline', `FINAL_HIDDEN[pos=${numTokens - 1}]: nan=${nanCount}/${data.length}, sample=[${nonZero.map(x => x.toFixed(4)).join(', ')}]`);
3713
- }
3714
- }
3715
-
3716
- if (hasGpuTimePrefill) {
3717
- this.#state.stats.gpuTimePrefillMs = gpuTimePrefillMs;
3718
- }
3719
-
3720
- if (returnHidden) {
3721
- if (currentRecorder) {
3722
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3723
- currentHiddenBuffer,
3724
- currentRecorder,
3725
- 'prefill_return_hidden_carry'
3726
- );
3727
- }
3728
- return {
3729
- numTokens,
3730
- config,
3731
- startPos,
3732
- activationDtype,
3733
- activationBytes,
3734
- currentRecorder,
3735
- recordProfile,
3736
- debugCheckBuffer,
3737
- currentHiddenBuffer,
3738
- };
3739
- }
3740
-
3741
-
3742
- let lastLogits;
3743
- let logitsVocabSize = config.vocabSize;
3744
- let usedRecordedLogits = false;
3745
- const lmHead = this.#state.weights.get('lm_head');
3746
- const canRecordLogits = !!currentRecorder
3747
- && selectedLogitTokenIds === null
3748
- && !!lmHead
3749
- && !isCpuWeightBuffer(lmHead)
3750
- && !this.#state.disableRecordedLogits
3751
- && numTokens === 1;
3752
- if (currentRecorder && canRecordLogits) {
3753
- const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
3754
- const recorded = await recordLogitsGPU(
3755
- currentRecorder,
3756
- currentHiddenBuffer,
3757
- numTokens,
3758
- getLogitsWeights(this.#state),
3759
- getLogitsConfig(this.#state),
3760
- this.#state.operatorDiagnostics
3761
- );
3762
- logitsVocabSize = recorded.vocabSize;
3763
- usedRecordedLogits = true;
3764
-
3765
- recordPrefillRecorderStats(this.#state, currentRecorder);
3766
- await currentRecorder.submitAndWait();
3767
- await recordProfile(currentRecorder);
3768
-
3769
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: recorded.logitsDtype });
3770
- const lastLogitsSize = logitsVocabSize * logitsBytes;
3771
- const lastLogitsOffset = (numTokens - 1) * lastLogitsSize;
3772
- const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
3773
- releaseBuffer(recorded.logitsBuffer);
3774
- lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
3775
- if (tracePrefillEnabled) {
3776
- trace.perf('Prefill recorded logits complete', {
3777
- numTokens,
3778
- vocabSize: logitsVocabSize,
3779
- elapsedMs: performance.now() - logitsTraceStart,
3780
- });
3781
- }
3782
-
3783
- const health = getLogitsHealth(lastLogits);
3784
- if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
3785
- log.warn(
3786
- 'Logits',
3787
- `Recorded logits invalid (nan=${health.nanCount} inf=${health.infCount} nonZero=${health.nonZeroCount}, maxAbs=${health.maxAbs.toFixed(3)}); recomputing without recorder.`
3788
- );
3789
- this.#state.disableRecordedLogits = true;
3790
- this.#state.disableFusedDecode = true;
3791
- const fallbackLogits = await computeLogits(
3792
- currentHiddenBuffer,
3793
- numTokens,
3794
- getLogitsWeights(this.#state),
3795
- getLogitsConfig(this.#state),
3796
- this.#state.useGPU,
3797
- this.#state.debugFlags,
3798
- undefined,
3799
- debugCheckBuffer,
3800
- this.#state.runtimeConfig.shared.debug.probes,
3801
- { lastPositionOnly: true },
3802
- this.#state.operatorDiagnostics
3803
- );
3804
- const fallbackHealth = getLogitsHealth(fallbackLogits);
3805
- if (fallbackHealth.nanCount > 0 || fallbackHealth.infCount > 0 || fallbackHealth.nonZeroCount === 0) {
3806
- throw new Error(
3807
- `[Logits] Fallback logits invalid (nan=${fallbackHealth.nanCount} inf=${fallbackHealth.infCount} nonZero=${fallbackHealth.nonZeroCount}, maxAbs=${fallbackHealth.maxAbs.toFixed(3)}). ` +
3808
- 'This indicates upstream kernel output is NaN/Inf (often prefill attention/matmul).'
3809
- );
3810
- }
3811
- logitsVocabSize = config.vocabSize;
3812
- usedRecordedLogits = false;
3813
- lastLogits = fallbackLogits.length === logitsVocabSize
3814
- ? fallbackLogits
3815
- : extractLastPositionLogits(fallbackLogits, numTokens, logitsVocabSize);
3816
- }
3817
-
3818
- releaseBuffer(currentHiddenBuffer);
3819
- } else {
3820
- const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
3821
- if (currentRecorder) {
3822
- currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3823
- currentHiddenBuffer,
3824
- currentRecorder,
3825
- 'prefill_logits_carry'
3826
- );
3827
- recordPrefillRecorderStats(this.#state, currentRecorder);
3828
- await currentRecorder.submitAndWait();
3829
- await recordProfile(currentRecorder);
3830
- }
3831
- const logits = await computeLogits(
3832
- currentHiddenBuffer,
3833
- numTokens,
3834
- getLogitsWeights(this.#state),
3835
- getLogitsConfig(this.#state),
3836
- this.#state.useGPU,
3837
- this.#state.debugFlags,
3838
- undefined,
3839
- debugCheckBuffer,
3840
- this.#state.runtimeConfig.shared.debug.probes,
3841
- { lastPositionOnly: true, selectedTokenIds: selectedLogitTokenIds },
3842
- this.#state.operatorDiagnostics
3843
- );
3844
-
3845
- if (selectedLogitTokenIds !== null) {
3846
- lastLogits = logits;
3847
- } else {
3848
- lastLogits = logits.length === logitsVocabSize
3849
- ? logits
3850
- : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
3851
- }
3852
- releaseBuffer(currentHiddenBuffer);
3853
- if (tracePrefillEnabled) {
3854
- trace.perf('Prefill logits complete', {
3855
- numTokens,
3856
- vocabSize: logitsVocabSize,
3857
- elapsedMs: performance.now() - logitsTraceStart,
3858
- });
3859
- }
3860
- }
3861
-
3862
- this.#state.currentSeqLen = startPos + numTokens;
3863
-
3864
- if (usedRecordedLogits) {
3865
- if (logitsVocabSize < config.vocabSize) {
3866
- const padded = new Float32Array(config.vocabSize);
3867
- padded.set(lastLogits);
3868
- padded.fill(-Infinity, logitsVocabSize);
3869
- lastLogits = padded;
3870
- }
3871
- if (config.finalLogitSoftcapping != null) {
3872
- applySoftcapping(lastLogits, config.finalLogitSoftcapping);
3873
- }
3874
- }
3875
-
3876
- if (opts.debug && selectedLogitTokenIds === null) {
3877
- logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
3878
- }
3879
- if (isTraceEnabled('logits')) {
3880
- trace.logits(selectedLogitTokenIds === null ? 'PREFILL_LOGITS_HEALTH' : 'PREFILL_SELECTED_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3881
- }
3882
-
3883
- if (opts.debug) {
3884
- if (this.#state.kvCache?.hasGPUCache?.()) {
3885
- log.debug('Pipeline', `KV cache active after prefill: seqLen=${this.#state.kvCache.getKeyCache(0)?.constructor.name ?? '?'}`);
3886
- } else {
3887
- log.warn('Pipeline', `KV cache NOT active after prefill! hasGPUCache=${this.#state.kvCache?.hasGPUCache?.()}`);
3888
- }
3889
- }
3890
-
3891
- if (tracePrefillEnabled) {
3892
- trace.perf('Prefill phase complete', {
3893
- numTokens,
3894
- totalMs: performance.now() - prefillTraceStart,
3895
- });
3896
- }
3897
-
3898
- return lastLogits;
581
+ return _prefillImpl.apply(this, arguments);
3899
582
  }
3900
583
 
3901
-
3902
584
  async _decodeStep(currentIds, opts) {
3903
- if (usesReplayPrefillDecode(this.#state)) {
3904
- const stepResult = await this._decodeStepToLogits(currentIds, opts);
3905
- return this._sampleNextTokenFromLogits(stepResult.logits, currentIds, opts);
3906
- }
3907
- const debugCheckBuffer = this.#state.debug
3908
- ? (buffer, label, numTokens, expectedDim) =>
3909
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
3910
- : undefined;
3911
- return decodeStep(this.#state, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
585
+ return _decodeStepImpl.apply(this, arguments);
3912
586
  }
3913
587
 
3914
588
  async decodeStepLogits(currentIds, options = {}) {
3915
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
3916
- if (this.#state.isGenerating && options.__internalGenerate !== true) {
3917
- throw new Error('Generation already in progress');
3918
- }
3919
- resetActiveExecutionPlan(this.#state);
3920
-
3921
- validateCallTimeOptions(options);
3922
-
3923
- const opts = this._resolveStepOptions(options);
3924
- return this._decodeStepToLogits(currentIds, opts);
589
+ return decodeStepLogitsImpl.apply(this, arguments);
3925
590
  }
3926
591
 
3927
592
  async advanceWithToken(tokenId, options = {}) {
3928
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
3929
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
3930
- resetActiveExecutionPlan(this.#state);
3931
- assertIncrementalDecodeSupport(this.#state, 'advanceWithToken');
3932
-
3933
- validateCallTimeOptions(options);
3934
-
3935
- const opts = this._resolveStepOptions(options);
3936
- const debugCheckBuffer = this.#state.debug
3937
- ? (buffer, label, numTokens, expectedDim) =>
3938
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
3939
- : undefined;
3940
-
3941
- this._assertTokenIdInRange(tokenId, 'advanceWithToken');
3942
- await advanceWithToken(this.#state, tokenId, opts, this._getDecodeHelpers(debugCheckBuffer));
593
+ return advanceWithTokenImpl.apply(this, arguments);
3943
594
  }
3944
595
 
3945
596
  async advanceWithTokenAndEmbedding(tokenId, options = {}) {
3946
- if (!this.#state.isLoaded) throw new Error('Model not loaded');
3947
- if (this.#state.isGenerating) throw new Error('Generation already in progress');
3948
- resetActiveExecutionPlan(this.#state);
3949
- assertIncrementalDecodeSupport(this.#state, 'advanceWithTokenAndEmbedding');
3950
-
3951
- validateCallTimeOptions(options);
3952
-
3953
- const opts = this._resolveStepOptions(options);
3954
- const embeddingMode = resolveAdvanceEmbeddingMode(this.#state, options);
3955
- const debugCheckBuffer = this.#state.debug
3956
- ? (buffer, label, numTokens, expectedDim) =>
3957
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
3958
- : undefined;
3959
-
3960
- this._assertTokenIdInRange(tokenId, 'advanceWithTokenAndEmbedding');
3961
- return runAdvanceWithTokenAndEmbedding(
3962
- this.#state,
3963
- tokenId,
3964
- opts,
3965
- this._getDecodeHelpers(debugCheckBuffer),
3966
- embeddingMode
3967
- );
597
+ return advanceWithTokenAndEmbeddingImpl.apply(this, arguments);
3968
598
  }
3969
599
 
3970
600
  async _generateNTokensGPU(startToken, N, currentIds, opts) {
3971
- const debugCheckBuffer = this.#state.debug
3972
- ? (buffer, label, numTokens, expectedDim) =>
3973
- debugCheckBufferHelper(this.#state, buffer, label, numTokens, expectedDim)
3974
- : undefined;
3975
- return generateNTokensGPU(this.#state, startToken, N, currentIds, opts, this._getDecodeHelpers(debugCheckBuffer));
601
+ return _generateNTokensGPUImpl.apply(this, arguments);
3976
602
  }
3977
603
  }