doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -15,19 +15,20 @@ import {
15
15
  isDecodeRecorderEnabled,
16
16
  isProfileDecodeRecorderEnabled,
17
17
  } from './execution-plan.js';
18
-
19
18
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
20
19
  import { isStopToken } from './init.js';
21
20
  import { embed } from './embed.js';
22
- import { resolvePerLayerInputsSession } from './generator-helpers.js';
21
+ import { resolvePerLayerInputsSession } from './generator/session-context.js';
23
22
  import { processLayer } from './layer.js';
24
- import { computeLogits, computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
23
+ import { computeLogits, computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, extractLastPositionLogits } from './logits/index.js';
25
24
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata } from '../../../gpu/weight-buffer.js';
26
25
  import { decodeReadback } from './debug-utils/index.js';
27
- import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-runtime.js';
26
+ import { captureObservedFusedDecodeLogits, emitObservedLogits } from './generator-logits-observation.js';
27
+ import { extractEmbeddingFromHiddenGPU } from './embedding-extraction.js';
28
28
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
29
29
  import { hasLinearAttentionLayers } from './linear-attention.js';
30
30
  import { hasConvLayers } from './layer.js';
31
+ import { advanceDecodeStepCount } from './tsir-fixture-writer.js';
31
32
  import {
32
33
  preparePerLayerInputs,
33
34
  createPleBufferCache,
@@ -36,22 +37,11 @@ import {
36
37
  hasGpuSplitPerLayerInputEmbeddings,
37
38
  getPleHotVocabularyRuntime,
38
39
  } from './per-layer-inputs.js';
40
+ import { FINITENESS_RESET_WORDS, FinitenessError, decodeStep, getEffectiveActivationDtype, isOwnedDecodeBuffer, recordDecodeProfileStep, releasePerLayerInputBuffer, runDecodeLayers, shouldLogProfileStep, sumProfileTimings } from './generator/decode.js';
41
+ export { FinitenessError, decodeStep, decodeStepLogits, readMappedBufferCopy, readSampledTokenFromStagingBuffer, shouldUseFusedDecodeSampling, sumProfileTimings } from './generator/decode.js';
39
42
 
40
- const UNKNOWN_TOKEN_TEXT = '<unknown>';
41
- const FINITENESS_RESET_WORDS = new Uint32Array(4);
42
43
  const FINITENESS_STATUS_BYTES = FINITENESS_RESET_WORDS.byteLength;
43
44
 
44
- export function sumProfileTimings(timings) {
45
- if (!timings || Object.keys(timings).length === 0) return null;
46
- let total = 0;
47
- for (const value of Object.values(timings)) {
48
- if (Number.isFinite(value)) {
49
- total += value;
50
- }
51
- }
52
- return total;
53
- }
54
-
55
45
  function mergeRecordOpLabelCounts(target, source) {
56
46
  const merged = target && typeof target === 'object' && !Array.isArray(target)
57
47
  ? target
@@ -69,136 +59,16 @@ function mergeRecordOpLabelCounts(target, source) {
69
59
  return merged;
70
60
  }
71
61
 
72
- function getEffectiveActivationDtype(state, opts) {
73
- const executionPlanDtype = opts?.executionPlan?.activationDtype;
74
- if (executionPlanDtype !== undefined && executionPlanDtype !== null) {
75
- return executionPlanDtype;
76
- }
77
- if (executionPlanDtype === null) {
78
- throw new Error('[Pipeline] executionPlan.activationDtype is required when provided and cannot be null.');
79
- }
80
- return state.runtimeConfig.inference.compute.activationDtype;
81
- }
82
-
83
- function getTokenTextOrUnknown(tokenizer, tokenId) {
84
- if (!tokenizer || typeof tokenizer.decode !== 'function') {
85
- return UNKNOWN_TOKEN_TEXT;
86
- }
87
-
88
- const tokenText = tokenizer.decode([tokenId], false, false);
89
- if (typeof tokenText !== 'string' || tokenText.length === 0) {
90
- return UNKNOWN_TOKEN_TEXT;
91
- }
92
-
93
- return tokenText;
94
- }
95
-
96
- function isOwnedDecodeBuffer(candidate, decodeHiddenBuffer, decodeAltBuffer) {
97
- if (candidate === decodeHiddenBuffer) {
98
- return true;
99
- }
100
- return candidate === decodeAltBuffer;
101
- }
102
-
103
- function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
104
- if (!buffer) {
105
- return;
106
- }
107
- const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
108
- if (ownsBuffer) {
109
- return;
110
- }
111
- const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
112
- if (cachedPleBuffer) {
113
- return;
114
- }
115
- if (recorder) {
116
- recorder.trackTemporaryBuffer(buffer);
117
- return;
118
- }
119
- releaseBuffer(buffer);
120
- }
121
-
122
- function schedulePlePrefetchForToken(state, tokenId) {
123
- if (state?.prefetchPleNextToken !== true) {
124
- return;
125
- }
126
- const config = state.modelConfig;
127
- const pleHiddenSize = Number(config?.hiddenSizePerLayerInput ?? 0);
128
- if (!Number.isFinite(pleHiddenSize) || pleHiddenSize <= 0) {
129
- return;
130
- }
131
- const pleWeights = state.weights.get('per_layer_inputs');
132
- if (!pleWeights?.embedTokensPerLayer) {
133
- return;
134
- }
135
- const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
136
- config.perLayerInputsSession ?? null,
137
- state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
138
- );
139
- state.plePrefetchPending = prefetchPerLayerRow(
140
- tokenId,
141
- pleWeights.embedTokensPerLayer,
142
- config.numLayers * pleHiddenSize,
143
- resolvedPerLayerInputsSession
144
- );
145
- }
146
-
147
- function getReusableSampleReadbackBuffer(state, device, size) {
148
- const existing = state.sampleReadbackBuffer;
149
- if (existing && existing.size >= size) {
150
- return existing;
151
- }
152
- if (existing) {
153
- existing.destroy();
154
- }
155
- const buffer = device.createBuffer({
156
- label: 'sample_staging_reuse',
157
- size,
158
- usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
159
- });
160
- state.sampleReadbackBuffer = buffer;
161
- return buffer;
162
- }
163
-
164
- export class FinitenessError extends Error {
165
- constructor(message) {
166
- super(message);
167
- this.name = 'FinitenessError';
168
- }
169
- }
170
-
171
- function shouldLogProfileStep(state, step) {
172
- const profilerConfig = state.runtimeConfig?.shared?.debug?.profiler;
173
- const every = profilerConfig?.logEveryDecodeSteps ?? 1;
174
- if (!Number.isFinite(every) || every <= 1) return true;
175
- return step === 1 || step % every === 0;
176
- }
177
-
178
- function recordDecodeProfileStep(state, entry) {
179
- if (!entry || (!entry.timings && !entry.recorderStats)) return;
180
- if (!state.stats.decodeProfileSteps) {
181
- state.stats.decodeProfileSteps = [];
182
- }
183
- state.stats.decodeProfileSteps.push(entry);
184
- }
185
-
186
62
  export function shouldUseBatchDecode(config) {
187
63
  return isBatchDecodeEnabled(config);
188
64
  }
189
65
 
190
- export function shouldUseFusedDecodeSampling(config) {
191
- return config.recorderEnabled === true
192
- && config.gpuSamplingEnabled === true
193
- && config.fusedDecodeDisabled !== true
194
- && !hasConvLayers(config.layerTypes ?? []);
195
- }
196
-
197
66
  function shouldUseGreedyLmHeadArgmaxFusion(state, opts, samplingDefaults, repetitionPenalty) {
198
67
  const probes = state.runtimeConfig?.shared?.debug?.probes;
199
- return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true
68
+ return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true && (state.modelConfig?.logitOutputScale ?? 1) === 1
200
69
  && opts.temperature < samplingDefaults.greedyThreshold
201
70
  && repetitionPenalty === 1.0
71
+ && opts.presencePenalty === 0
202
72
  && state.operatorDiagnostics == null
203
73
  && !(Array.isArray(probes) && probes.length > 0);
204
74
  }
@@ -276,72 +146,6 @@ export function findInvalidGeneratedToken(tokens, vocabSize, padTokenId = null)
276
146
  return null;
277
147
  }
278
148
 
279
- export async function readSampledTokenFromStagingBuffer(stagingBuffer, options = {}) {
280
- const ownsStagingBuffer = options.ownsStagingBuffer === true;
281
- const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
282
- const ring = options.ring ?? null;
283
- const cleanupRecorder = options.cleanupRecorder ?? null;
284
- const timing = {
285
- mapWaitMs: 0,
286
- cleanupMs: 0,
287
- copyMs: 0,
288
- };
289
- let mapped = false;
290
- let cleanupCompleted = false;
291
-
292
- try {
293
- const mapStart = performance.now();
294
- await stagingBuffer.mapAsync(GPUMapMode.READ);
295
- timing.mapWaitMs = performance.now() - mapStart;
296
- mapped = true;
297
- const cleanupStart = performance.now();
298
- await cleanupRecorder?.completeDeferredCleanup();
299
- timing.cleanupMs = performance.now() - cleanupStart;
300
- cleanupCompleted = true;
301
- const copyStart = performance.now();
302
- const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
303
- const result = {
304
- nextToken: mappedWords[0],
305
- finitenessStatus: hasFinitenessBuffer
306
- ? parseFinitenessStatusWords(mappedWords, 1)
307
- : parseFinitenessStatusWords(mappedWords, 0),
308
- timing,
309
- };
310
- timing.copyMs = performance.now() - copyStart;
311
- return result;
312
- } finally {
313
- if (mapped) {
314
- stagingBuffer.unmap();
315
- }
316
- if (!cleanupCompleted) {
317
- await cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
318
- }
319
- if (ownsStagingBuffer) {
320
- stagingBuffer.destroy();
321
- }
322
- ring?.advance();
323
- }
324
- }
325
-
326
-
327
- export async function readMappedBufferCopy(stagingBuffer, options = {}) {
328
- const ownsStagingBuffer = options.ownsStagingBuffer !== false;
329
- let mapped = false;
330
-
331
- try {
332
- await stagingBuffer.mapAsync(GPUMapMode.READ);
333
- mapped = true;
334
- return stagingBuffer.getMappedRange().slice(0);
335
- } finally {
336
- if (mapped) {
337
- stagingBuffer.unmap();
338
- }
339
- if (ownsStagingBuffer) {
340
- stagingBuffer.destroy();
341
- }
342
- }
343
- }
344
-
345
149
  export async function readBatchTokensFromStagingBuffers(options) {
346
150
  const {
347
151
  tokensStagingBuffer,
@@ -440,730 +244,8 @@ export async function readBatchTokensFromStagingBuffers(options) {
440
244
  }
441
245
  }
442
246
 
443
- async function runDecodeLayers(state, tokenId, opts, helpers) {
444
- const config = state.modelConfig;
445
- const debugCheckBuffer = state.debug ? helpers.debugCheckBuffer : undefined;
446
-
447
- const context = helpers.buildLayerContext(undefined, true, opts.debugLayers, opts.executionPlan);
448
- context.currentTokenIds = [tokenId];
449
-
450
- state.decodeBuffers.resetPingPong();
451
-
452
- const decodeHiddenBuffer = state.decodeBuffers.getHiddenBuffer();
453
- const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
454
-
455
- const embedBufferRaw = state.weights.get('embed');
456
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
457
- throw new Error('Embed buffer not found or not a supported buffer type');
458
- }
459
- const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
460
- const embedDtype = isCpuWeightBuffer(embedBufferRaw)
461
- ? embedBufferRaw.dtype
462
- : getWeightDtype(embedBufferRaw);
463
- const embedMetadata = getWeightMetadata(embedBufferRaw);
464
- const activationDtype = getEffectiveActivationDtype(state, opts);
465
-
466
- const embedTensor = await embed([tokenId], embedBuffer, {
467
- hiddenSize: config.hiddenSize,
468
- vocabSize: config.vocabSize,
469
- scaleEmbeddings: config.scaleEmbeddings,
470
- embeddingScale: config.embeddingScale,
471
- outputBuffer: decodeHiddenBuffer ?? undefined,
472
- transpose: state.embeddingTranspose,
473
- debugProbes: state.runtimeConfig.shared.debug.probes,
474
- operatorDiagnostics: state.operatorDiagnostics,
475
- activationDtype,
476
- embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
477
- embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
478
- executionPolicies: state.executionV1State?.policies ?? null,
479
- });
480
-
481
- let hiddenStates = embedTensor.buffer;
482
-
483
- // Resolve pending PLE prefetch from previous decode step
484
- let plePrefetchResult = null;
485
- if (state.plePrefetchPending) {
486
- plePrefetchResult = await state.plePrefetchPending;
487
- state.plePrefetchPending = null;
488
- }
489
-
490
- const perLayerInputs = await preparePerLayerInputs([tokenId], embedTensor, context, {
491
- numTokens: 1,
492
- pleCache: state.pleCache ?? null,
493
- prefetchedRow: plePrefetchResult,
494
- });
495
-
496
- try {
497
- for (let l = 0; l < config.numLayers; l++) {
498
- context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
499
- const prevStates = hiddenStates;
500
- hiddenStates = (await processLayer(l, hiddenStates, 1, false, context));
501
- state.decodeBuffers.swapPingPong();
502
- releasePerLayerInputBuffer(context.perLayerInputBuffer, null, context.decodeBuffers, state.pleCache ?? null);
503
- if (perLayerInputs) {
504
- perLayerInputs[l] = null;
505
- }
506
- context.perLayerInputBuffer = null;
507
- if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
508
- const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
509
- if (!isPreAllocated) {
510
- releaseBuffer(prevStates);
511
- }
512
- }
513
- }
514
- } finally {
515
- context.perLayerInputBuffer = null;
516
- if (perLayerInputs) {
517
- for (const buffer of perLayerInputs) {
518
- releasePerLayerInputBuffer(buffer, null, context.decodeBuffers, state.pleCache ?? null);
519
- }
520
- }
521
- helpers.releaseSharedAttentionState?.(context.sharedAttentionState, null);
522
- }
523
-
524
- return { hiddenStates, decodeHiddenBuffer, decodeAltBuffer, debugCheckBuffer, context };
525
- }
526
-
527
- function createDecodeRecorder(state, opts) {
528
- const device = getDevice();
529
- const executionPlan = opts.executionPlan;
530
- const recorderConfig = {
531
- hasDevice: Boolean(device),
532
- debug: opts.debug,
533
- disableCommandBatching: executionPlan?.disableCommandBatching ?? opts.disableCommandBatching,
534
- kvLayout: state.kvCache?.layout ?? null,
535
- };
536
- const recorderEnabled = opts.profile
537
- ? isProfileDecodeRecorderEnabled(recorderConfig)
538
- : isDecodeRecorderEnabled(recorderConfig);
539
- let recorder;
540
- if (recorderEnabled) {
541
- recorder = opts.profile
542
- ? createProfilingRecorder('decode', device)
543
- : createCommandRecorder('decode', { recordLabels: opts.debug === true || opts.benchmark === true }, device);
544
- }
545
- if (state.decodeStepCount === 1) {
546
- const path = selectRuleValue('inference', 'config', 'tracePath', { useRecorder: Boolean(recorder) });
547
- log.debug('Decode', `Using ${path} path (recorder=${!!recorder}, debug=${opts.debug})`);
548
- }
549
- return recorder;
550
- }
551
-
552
- async function submitDecodeRecorderProfile(state, opts, recorder, profileLabel) {
553
- if (!recorder) {
554
- return;
555
- }
556
- await recorder.submitAndWait();
557
-
558
- if (!opts.profile) {
559
- return;
560
- }
561
-
562
- const timings = recorder.isProfilingEnabled()
563
- ? await recorder.resolveProfileTimings()
564
- : null;
565
- const total = sumProfileTimings(timings);
566
- if (total !== null) {
567
- state.stats.gpuTimeDecodeMs = (state.stats.gpuTimeDecodeMs ?? 0) + total;
568
- }
569
- if (timings || opts.executionObserver) {
570
- recordDecodeProfileStep(state, {
571
- step: state.decodeStepCount,
572
- timings,
573
- totalMs: total ?? undefined,
574
- recorderStats: recorder.getStats(),
575
- });
576
- if (!opts.executionObserver && shouldLogProfileStep(state, state.decodeStepCount)) {
577
- log.warn('Profile', `Decode step ${state.decodeStepCount}${profileLabel}:`);
578
- log.warn('Profile', CommandRecorder.formatProfileReport(timings));
579
- }
580
- }
581
- }
582
-
583
- export async function decodeStep(state, currentIds, opts, helpers) {
584
- const stepWallStart = performance.now();
585
- const lastToken = currentIds[currentIds.length - 1];
586
- const numTokens = 1;
587
- const config = state.modelConfig;
588
- const samplingDefaults = state.runtimeConfig.inference.sampling;
589
- const executionPlan = opts.executionPlan;
590
- const debugCheckBuffer = state.debug ? helpers.debugCheckBuffer : undefined;
591
-
592
- state.decodeStepCount++;
593
- const isDebugStep = opts.debug && state.decodeStepCount <= 5;
594
- if (isDebugStep) {
595
- const tokenText = getTokenTextOrUnknown(state.tokenizer, lastToken);
596
- log.debug('Decode', `[${state.decodeStepCount}] token="${tokenText}" pos=${state.currentSeqLen}`);
597
- }
598
-
599
- const device = getDevice();
600
- const recorder = createDecodeRecorder(state, opts);
601
-
602
- if (state.finitenessBuffer && device) {
603
- device.queue.writeBuffer(state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
604
- }
605
-
606
- const context = helpers.buildLayerContext(recorder, true, opts.debugLayers, executionPlan);
607
- context.currentTokenIds = [lastToken];
608
-
609
- state.decodeBuffers.resetPingPong();
610
-
611
- const decodeHiddenBuffer = state.decodeBuffers.getHiddenBuffer();
612
- const decodeAltBuffer = state.decodeBuffers.getOutputHiddenBuffer();
613
-
614
- const embedBufferRaw = state.weights.get('embed');
615
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
616
- throw new Error('Embed buffer not found or not a supported buffer type');
617
- }
618
- const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
619
- const embedDtype = isCpuWeightBuffer(embedBufferRaw)
620
- ? embedBufferRaw.dtype
621
- : getWeightDtype(embedBufferRaw);
622
- const embedMetadata = getWeightMetadata(embedBufferRaw);
623
- const activationDtype = getEffectiveActivationDtype(state, opts);
624
- const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
625
-
626
- const embedTensor = await embed([lastToken], embedBuffer, {
627
- hiddenSize: config.hiddenSize,
628
- vocabSize: config.vocabSize,
629
- scaleEmbeddings: config.scaleEmbeddings,
630
- embeddingScale: config.embeddingScale,
631
- recorder,
632
- outputBuffer: decodeHiddenBuffer ?? undefined,
633
- transpose: state.embeddingTranspose,
634
- debugProbes: state.runtimeConfig.shared.debug.probes,
635
- operatorDiagnostics: state.operatorDiagnostics,
636
- activationDtype,
637
- embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
638
- embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
639
- executionPolicies: state.executionV1State?.policies ?? null,
640
- });
641
-
642
- let hiddenStates = embedTensor.buffer;
643
-
644
- // Resolve pending PLE prefetch from previous decode step
645
- let plePrefetchResult = null;
646
- if (state.plePrefetchPending) {
647
- plePrefetchResult = await state.plePrefetchPending;
648
- state.plePrefetchPending = null;
649
- }
650
-
651
- const perLayerInputs = await preparePerLayerInputs([lastToken], embedTensor, context, {
652
- numTokens: 1,
653
- pleCache: state.pleCache ?? null,
654
- prefetchedRow: plePrefetchResult,
655
- });
656
-
657
- if (opts.debug && state.decodeStepCount === 1) {
658
- const validSize = config.hiddenSize * activationBytes;
659
- const embedData = await readBuffer(hiddenStates, validSize);
660
- const embedArr = decodeReadback(embedData, activationDtype);
661
- const sample = embedArr.slice(0, 5);
662
- const maxAbs = Math.max(...embedArr.map(Math.abs));
663
- const nonZero = embedArr.filter(x => Math.abs(x) > 1e-10).length;
664
- log.debug('Decode', `[1] Embed check: maxAbs=${maxAbs.toFixed(2)}, nonZero=${nonZero}/${embedArr.length}, sample=[${Array.from(sample).map(v => v.toFixed(3)).join(', ')}]`);
665
- }
666
-
667
- const benchmarkSubmits = state.decodeStepCount <= 3 && opts.debug;
668
- if (benchmarkSubmits) {
669
- setTrackSubmits(true);
670
- resetSubmitStats();
671
- }
672
-
673
- const hasGPUCache = context.kvCache?.hasGPUCache?.() ?? false;
674
- if (opts.debug && state.decodeStepCount === 1) {
675
- log.debug('Decode', `KV cache check: hasGPUCache=${hasGPUCache}, currentSeqLen=${context.currentSeqLen}`);
676
- }
677
-
678
- try {
679
- for (let l = 0; l < config.numLayers; l++) {
680
- context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
681
- const prevStates = hiddenStates;
682
- hiddenStates = (await processLayer(l, hiddenStates, numTokens, false, context));
683
-
684
- state.decodeBuffers.swapPingPong();
685
- releasePerLayerInputBuffer(
686
- context.perLayerInputBuffer,
687
- recorder,
688
- context.decodeBuffers,
689
- state.pleCache ?? null
690
- );
691
- if (perLayerInputs) {
692
- perLayerInputs[l] = null;
693
- }
694
- context.perLayerInputBuffer = null;
695
-
696
- if (isGpuBufferInstance(prevStates) && prevStates !== hiddenStates) {
697
- const isPreAllocated = isOwnedDecodeBuffer(prevStates, decodeHiddenBuffer, decodeAltBuffer);
698
- if (!isPreAllocated) {
699
- if (recorder) {
700
- recorder.trackTemporaryBuffer(prevStates);
701
- } else {
702
- releaseBuffer(prevStates);
703
- }
704
- }
705
- }
706
- }
707
- } finally {
708
- context.perLayerInputBuffer = null;
709
- if (perLayerInputs) {
710
- for (const buffer of perLayerInputs) {
711
- releasePerLayerInputBuffer(
712
- buffer,
713
- recorder,
714
- context.decodeBuffers,
715
- state.pleCache ?? null
716
- );
717
- }
718
- }
719
- helpers.releaseSharedAttentionState?.(context.sharedAttentionState, recorder);
720
- }
721
-
722
- const logitSoftcap = config.finalLogitSoftcapping === null
723
- ? 0
724
- : config.finalLogitSoftcapping;
725
- const padTokenId = state.tokenizer?.getSpecialTokens?.()?.pad ?? null;
726
- const lmHeadIsCpu = isCpuWeightBuffer(state.weights.get('lm_head'));
727
- const useGPUSampling = state.useGPU && isGPUSamplingAvailable() && !lmHeadIsCpu;
728
- const useFusedDecode = shouldUseFusedDecodeSampling({
729
- recorderEnabled: Boolean(recorder),
730
- gpuSamplingEnabled: useGPUSampling,
731
- fusedDecodeDisabled: state.disableFusedDecode,
732
- layerTypes: config.layerTypes,
733
- });
734
-
735
- if (useFusedDecode) {
736
- const ring = state.decodeRing;
737
- let ringSlot = null;
738
- if (ring) {
739
- ring.ensure({
740
- batchSize: 1,
741
- tokensPerInterval: 1,
742
- stopCheckMode: executionPlan?.stopCheckMode ?? opts.stopCheckMode,
743
- ringTokens: executionPlan?.ringTokens ?? state.runtimeConfig.inference.batching.ringTokens,
744
- ringStop: executionPlan?.ringStop ?? state.runtimeConfig.inference.batching.ringStop,
745
- ringStaging: executionPlan?.ringStaging ?? state.runtimeConfig.inference.batching.ringStaging,
746
- });
747
- ringSlot = ring.acquire();
748
- }
749
-
750
- const { logitsBuffer, vocabSize, logitsDtype } = await recordLogitsGPU(
751
- recorder,
752
- hiddenStates,
753
- numTokens,
754
- helpers.getLogitsWeights(),
755
- helpers.getLogitsConfig(),
756
- state.operatorDiagnostics,
757
- );
758
-
759
- const ringTokensBuffer = ringSlot?.tokens ?? null;
760
- const sampleOutputBuffer = opts.temperature < samplingDefaults.greedyThreshold
761
- ? await recordArgmax(recorder, logitsBuffer, vocabSize, {
762
- padTokenId,
763
- logitSoftcap,
764
- logitsDtype,
765
- outputBuffer: ringTokensBuffer ?? undefined,
766
- outputIndex: 0,
767
- })
768
- : await recordGPUSample(recorder, logitsBuffer, vocabSize, {
769
- temperature: opts.temperature,
770
- topK: opts.topK,
771
- padTokenId,
772
- logitSoftcap,
773
- logitsDtype,
774
- outputBuffer: ringTokensBuffer ?? undefined,
775
- outputIndex: 0,
776
- greedyThreshold: samplingDefaults.greedyThreshold,
777
- randomSeed: opts.seed,
778
- });
779
-
780
- const ringStagingBuffer = ringSlot?.stagingTokens ?? null;
781
- const stagingSize = state.finitenessBuffer ? 20 : 4;
782
- const stagingBuffer = ringStagingBuffer && ringStagingBuffer.size >= stagingSize
783
- ? ringStagingBuffer
784
- : getReusableSampleReadbackBuffer(state, device, stagingSize);
785
- const ownsStagingBuffer = false;
786
- const ownsSampleOutputBuffer = !ringTokensBuffer || sampleOutputBuffer !== ringTokensBuffer;
787
-
788
- const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
789
- const encoder = recorder.getEncoder();
790
- encoder.copyBufferToBuffer(sampleOutputBuffer, 0, stagingBuffer, 0, 4);
791
- if (state.finitenessBuffer) {
792
- encoder.copyBufferToBuffer(state.finitenessBuffer, 0, stagingBuffer, 4, 16);
793
- }
794
-
795
- const readbackMode = executionPlan?.readbackMode;
796
- const isOverlapped = readbackMode === 'overlapped';
797
-
798
- // In overlapped mode, advance ring BEFORE submit so the GPU's next copy
799
- // target is a fresh slot while we read the current one.
800
- if (isOverlapped) {
801
- ring?.advance();
802
- }
803
-
804
- if (!allowReadback('pipeline.decode.sample')) {
805
- throw new Error('[Pipeline] GPU readback disabled for sampling');
806
- }
807
-
808
- const submitStart = performance.now();
809
- recorder.submit({ cleanup: 'deferred' });
810
- const submitWaitMs = performance.now() - submitStart;
811
-
812
- const readbackStart = performance.now();
813
- const readbackResult = await readSampledTokenFromStagingBuffer(stagingBuffer, {
814
- ownsStagingBuffer,
815
- hasFinitenessBuffer: Boolean(state.finitenessBuffer),
816
- ring: isOverlapped ? null : ring,
817
- cleanupRecorder: recorder,
818
- });
819
- const readbackWaitMs = performance.now() - readbackStart;
820
-
821
- state.stats.singleTokenSubmitWaitMs = (state.stats.singleTokenSubmitWaitMs ?? 0) + submitWaitMs;
822
- state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + readbackWaitMs;
823
- if (readbackResult.timing) {
824
- state.stats.singleTokenReadbackMapWaitMs = (state.stats.singleTokenReadbackMapWaitMs ?? 0)
825
- + readbackResult.timing.mapWaitMs;
826
- state.stats.singleTokenReadbackCleanupMs = (state.stats.singleTokenReadbackCleanupMs ?? 0)
827
- + readbackResult.timing.cleanupMs;
828
- state.stats.singleTokenReadbackCopyMs = (state.stats.singleTokenReadbackCopyMs ?? 0)
829
- + readbackResult.timing.copyMs;
830
- }
831
-
832
- const { nextToken: fusedNextToken, finitenessStatus } = readbackResult;
833
-
834
- if (finitenessStatus.triggered) {
835
- releaseBuffer(logitsBuffer);
836
- if (ownsSampleOutputBuffer) releaseBuffer(sampleOutputBuffer);
837
- if (!isPreAllocated) releaseBuffer(hiddenStates);
838
- throw new FinitenessError(`F16 bounds exceeded during generation${finitenessStatus.metadata}`);
839
- }
840
-
841
- log.debug('Decode', `Step ${state.decodeStepCount}: token=${fusedNextToken} (vocabSize=${config.vocabSize})`);
842
-
843
- const invalidToken = fusedNextToken >= config.vocabSize
844
- || (padTokenId != null && fusedNextToken === padTokenId)
845
- || (padTokenId == null && fusedNextToken === 0);
846
- if (!invalidToken) {
847
- schedulePlePrefetchForToken(state, fusedNextToken);
848
- }
849
- if (invalidToken) {
850
- log.warn('Decode', `Suspicious token ${fusedNextToken} (vocabSize=${config.vocabSize}, step=${state.decodeStepCount})`);
851
- if (allowReadback('pipeline.decode.debug-logits')) {
852
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
853
- const logitSample = await readBuffer(logitsBuffer, Math.min(config.vocabSize * logitsBytes, 4096));
854
- const logitArr = decodeReadback(logitSample, logitsDtype);
855
- const maxLogit = Math.max(...logitArr);
856
- const minLogit = Math.min(...logitArr);
857
- const hasNaN = logitArr.some((v) => Number.isNaN(v));
858
- const hasInf = logitArr.some((v) => !Number.isFinite(v));
859
- let argmaxIdx = 0;
860
- let argmaxVal = logitArr[0];
861
- for (let i = 1; i < logitArr.length; i++) {
862
- if (logitArr[i] > argmaxVal) {
863
- argmaxVal = logitArr[i];
864
- argmaxIdx = i;
865
- }
866
- }
867
- log.warn('Decode', `Logits: max=${maxLogit.toFixed(4)} at [${argmaxIdx}], min=${minLogit.toFixed(4)}, hasNaN=${hasNaN}, hasInf=${hasInf}`);
868
- log.warn('Decode', `First 10 logits: ${Array.from(logitSample.slice(0, 10)).map((v) => v.toFixed(4)).join(', ')}`);
869
- log.warn('Decode', `Logit[0] (pad): ${logitArr[0].toFixed(4)}, Logit[${argmaxIdx}]: ${argmaxVal.toFixed(4)}`);
870
- }
871
- }
872
-
873
- releaseBuffer(logitsBuffer);
874
- if (ownsSampleOutputBuffer) {
875
- releaseBuffer(sampleOutputBuffer);
876
- }
877
-
878
- if (benchmarkSubmits) {
879
- logSubmitStats(`Decode step ${state.decodeStepCount} (${config.numLayers} layers, fused)`);
880
- setTrackSubmits(false);
881
- }
882
-
883
- if (opts.profile) {
884
- const timings = recorder.isProfilingEnabled()
885
- ? await recorder.resolveProfileTimings()
886
- : null;
887
- const total = sumProfileTimings(timings);
888
- if (total !== null) {
889
- state.stats.gpuTimeDecodeMs = (state.stats.gpuTimeDecodeMs ?? 0) + total;
890
- }
891
- if (timings || opts.executionObserver) {
892
- recordDecodeProfileStep(state, {
893
- step: state.decodeStepCount,
894
- timings,
895
- totalMs: total ?? undefined,
896
- recorderStats: recorder.getStats(),
897
- });
898
- if (!opts.executionObserver && shouldLogProfileStep(state, state.decodeStepCount)) {
899
- log.warn('Profile', `Decode step ${state.decodeStepCount}:`);
900
- log.warn('Profile', CommandRecorder.formatProfileReport(timings));
901
- }
902
- }
903
- }
904
-
905
- if (invalidToken) {
906
- state.disableFusedDecode = true;
907
- log.warn('Decode', 'Fused sampling produced invalid token; falling back to CPU sampling.');
908
- const fallbackLogits = await computeLogits(
909
- hiddenStates,
910
- numTokens,
911
- helpers.getLogitsWeights(),
912
- helpers.getLogitsConfig(),
913
- state.useGPU,
914
- state.debugFlags,
915
- undefined,
916
- debugCheckBuffer,
917
- state.runtimeConfig.shared.debug.probes,
918
- null,
919
- state.operatorDiagnostics
920
- );
921
- applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty);
922
- const fallbackToken = sample(fallbackLogits, {
923
- temperature: opts.temperature,
924
- topP: opts.topP,
925
- topK: opts.topK,
926
- padTokenId,
927
- seed: opts.seed,
928
- });
929
- schedulePlePrefetchForToken(state, fallbackToken);
930
- if (!isPreAllocated) {
931
- releaseBuffer(hiddenStates);
932
- }
933
- state.currentSeqLen++;
934
- return fallbackToken;
935
- }
936
-
937
- if (!isPreAllocated) {
938
- releaseBuffer(hiddenStates);
939
- }
940
-
941
- state.currentSeqLen++;
942
- const stepWallMs = performance.now() - stepWallStart;
943
- state.stats.singleTokenOrchestrationMs = (state.stats.singleTokenOrchestrationMs ?? 0)
944
- + Math.max(0, stepWallMs - submitWaitMs - readbackWaitMs);
945
- return fusedNextToken;
946
- }
947
-
948
- await submitDecodeRecorderProfile(state, opts, recorder, ' (layers only)');
949
-
950
- if (benchmarkSubmits) {
951
- logSubmitStats(`Decode step ${state.decodeStepCount} (${config.numLayers} layers)`);
952
- setTrackSubmits(false);
953
- }
954
-
955
- if (opts.debug && state.decodeStepCount === 1 && isGpuBufferInstance(hiddenStates)) {
956
- const debugDevice = getDevice();
957
- if (debugDevice) {
958
- if (allowReadback('pipeline.decode.debug-hidden')) {
959
- const debugReadbackSize = state.runtimeConfig.shared.debug.pipeline.readbackSampleSize;
960
- const sampleSize = Math.min(debugReadbackSize, hiddenStates.size);
961
- const staging = debugDevice.createBuffer({
962
- size: sampleSize,
963
- usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
964
- });
965
- const enc = debugDevice.createCommandEncoder();
966
- enc.copyBufferToBuffer(hiddenStates, 0, staging, 0, sampleSize);
967
- debugDevice.queue.submit([enc.finish()]);
968
- const data = new Float32Array(await readMappedBufferCopy(staging));
969
- const nanCount = Array.from(data).filter(x => !Number.isFinite(x)).length;
970
- const nonZero = Array.from(data).filter(x => Number.isFinite(x) && x !== 0).slice(0, 5);
971
- log.debug('Decode', `[1] HIDDEN_AFTER_LAYERS: nan=${nanCount}/${data.length}, nonZero=${nonZero.length}, sample=[${nonZero.map(x => x.toFixed(4)).join(', ')}]`);
972
- }
973
- }
974
- }
975
-
976
- if (useGPUSampling) {
977
- const logitsResult = await computeLogitsGPU(
978
- hiddenStates,
979
- numTokens,
980
- helpers.getLogitsWeights(),
981
- helpers.getLogitsConfig(),
982
- state.debugFlags,
983
- state.operatorDiagnostics
984
- );
985
- if (logitsResult) {
986
- const { logitsBuffer, vocabSize, logitsDtype } = logitsResult;
987
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
988
- const nfReadbackStart = performance.now();
989
- const logitsData = await readBuffer(logitsBuffer, numTokens * vocabSize * logitsBytes);
990
- const nfReadbackMs = performance.now() - nfReadbackStart;
991
- state.stats.singleTokenReadbackWaitMs = (state.stats.singleTokenReadbackWaitMs ?? 0) + nfReadbackMs;
992
- releaseBuffer(logitsBuffer);
993
-
994
- const rawLogits = decodeReadback(logitsData, logitsDtype);
995
- const finalizedLogits = await finalizeLogits(
996
- rawLogits,
997
- numTokens,
998
- vocabSize,
999
- config.vocabSize,
1000
- config,
1001
- state.runtimeConfig.shared.debug.probes,
1002
- state.operatorDiagnostics
1003
- );
1004
- const sampledLogits = extractLastPositionLogits(finalizedLogits, numTokens, config.vocabSize);
1005
-
1006
- applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty);
1007
- const nextToken = sample(sampledLogits, {
1008
- temperature: opts.temperature,
1009
- topP: opts.topP,
1010
- topK: opts.topK,
1011
- padTokenId,
1012
- seed: opts.seed,
1013
- });
1014
-
1015
- if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
1016
- releaseBuffer(hiddenStates);
1017
- }
1018
- state.currentSeqLen++;
1019
- const nfStepWallMs = performance.now() - stepWallStart;
1020
- state.stats.singleTokenOrchestrationMs = (state.stats.singleTokenOrchestrationMs ?? 0)
1021
- + Math.max(0, nfStepWallMs - nfReadbackMs);
1022
- return nextToken;
1023
- }
1024
- }
1025
-
1026
- if (state.finitenessBuffer) {
1027
- const isInfiniteData = await readBuffer(state.finitenessBuffer, 16);
1028
- const u32 = new Uint32Array(isInfiniteData.buffer, isInfiniteData.byteOffset, 4);
1029
- const finitenessStatus = parseFinitenessStatusWords(u32, 0);
1030
- if (finitenessStatus.triggered) {
1031
- if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
1032
- releaseBuffer(hiddenStates);
1033
- }
1034
- throw new FinitenessError(`F16 bounds exceeded during generation${finitenessStatus.metadata}`);
1035
- }
1036
- }
1037
-
1038
- const logits = await computeLogits(
1039
- hiddenStates,
1040
- numTokens,
1041
- helpers.getLogitsWeights(),
1042
- helpers.getLogitsConfig(),
1043
- state.useGPU,
1044
- state.debugFlags,
1045
- undefined,
1046
- debugCheckBuffer,
1047
- state.runtimeConfig.shared.debug.probes,
1048
- null,
1049
- state.operatorDiagnostics
1050
- );
1051
-
1052
- if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
1053
- releaseBuffer(hiddenStates);
1054
- }
1055
-
1056
- if (isDebugStep) {
1057
- logitsSanity(logits, `Decode[${state.decodeStepCount}]`, opts.decode);
1058
- }
1059
-
1060
- applyRepetitionPenalty(logits, currentIds, opts.repetitionPenalty);
1061
- const nextToken = sample(logits, {
1062
- temperature: opts.temperature,
1063
- topP: opts.topP,
1064
- topK: opts.topK,
1065
- padTokenId,
1066
- seed: opts.seed,
1067
- });
1068
-
1069
- state.currentSeqLen++;
1070
- const cpuStepWallMs = performance.now() - stepWallStart;
1071
- state.stats.singleTokenOrchestrationMs = (state.stats.singleTokenOrchestrationMs ?? 0) + cpuStepWallMs;
1072
- return nextToken;
1073
- }
1074
-
1075
- export async function decodeStepLogits(state, currentIds, opts, helpers) {
1076
- const lastToken = currentIds[currentIds.length - 1];
1077
- const numTokens = 1;
1078
- const config = state.modelConfig;
1079
-
1080
- state.decodeStepCount++;
1081
- const recorder = createDecodeRecorder(state, opts);
1082
-
1083
- const { hiddenStates, decodeHiddenBuffer, decodeAltBuffer, debugCheckBuffer } = await runDecodeLayers(
1084
- state,
1085
- lastToken,
1086
- opts,
1087
- {
1088
- ...helpers,
1089
- buildLayerContext: (ignoredRecorder, isDecode, debugLayers, executionPlan) =>
1090
- helpers.buildLayerContext(recorder, isDecode, debugLayers, executionPlan),
1091
- }
1092
- );
1093
-
1094
- await submitDecodeRecorderProfile(state, opts, recorder, ' (layers only)');
1095
-
1096
- let logitsBuffer = null;
1097
- let logitsDtype = null;
1098
- let rawVocabSize = config.vocabSize;
1099
- let logits = null;
1100
-
1101
- if (state.useGPU && !isCpuWeightBuffer(state.weights.get('lm_head'))) {
1102
- const logitsResult = await computeLogitsGPU(
1103
- hiddenStates,
1104
- numTokens,
1105
- helpers.getLogitsWeights(),
1106
- helpers.getLogitsConfig(),
1107
- state.debugFlags,
1108
- state.operatorDiagnostics
1109
- );
1110
-
1111
- if (logitsResult) {
1112
- logitsBuffer = logitsResult.logitsBuffer;
1113
- logitsDtype = logitsResult.logitsDtype;
1114
- rawVocabSize = logitsResult.vocabSize;
1115
-
1116
- const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
1117
- const logitsData = await readBuffer(logitsBuffer, numTokens * rawVocabSize * logitsBytes);
1118
- const rawLogits = decodeReadback(logitsData, logitsDtype);
1119
- const finalized = await finalizeLogits(
1120
- rawLogits,
1121
- numTokens,
1122
- rawVocabSize,
1123
- config.vocabSize,
1124
- config,
1125
- state.runtimeConfig.shared.debug.probes,
1126
- state.operatorDiagnostics
1127
- );
1128
- logits = extractLastPositionLogits(finalized, numTokens, config.vocabSize);
1129
- }
1130
- }
1131
-
1132
- if (!logits) {
1133
- const rawLogits = await computeLogits(
1134
- hiddenStates,
1135
- numTokens,
1136
- helpers.getLogitsWeights(),
1137
- helpers.getLogitsConfig(),
1138
- state.useGPU,
1139
- state.debugFlags,
1140
- undefined,
1141
- debugCheckBuffer,
1142
- state.runtimeConfig.shared.debug.probes,
1143
- null,
1144
- state.operatorDiagnostics
1145
- );
1146
- logits = extractLastPositionLogits(rawLogits, numTokens, config.vocabSize);
1147
- }
1148
-
1149
- const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
1150
- if (!isPreAllocated) {
1151
- releaseBuffer(hiddenStates);
1152
- }
1153
-
1154
- state.currentSeqLen++;
1155
-
1156
- return {
1157
- logits,
1158
- logitsBuffer,
1159
- logitsDtype,
1160
- rawVocabSize,
1161
- vocabSize: config.vocabSize,
1162
- };
1163
- }
1164
-
1165
247
  export async function advanceWithToken(state, tokenId, opts, helpers) {
1166
- state.decodeStepCount++;
248
+ advanceDecodeStepCount(state);
1167
249
 
1168
250
  const { hiddenStates, decodeHiddenBuffer, decodeAltBuffer } = await runDecodeLayers(
1169
251
  state,
@@ -1182,7 +264,7 @@ export async function advanceWithToken(state, tokenId, opts, helpers) {
1182
264
 
1183
265
  export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers, embeddingMode) {
1184
266
 
1185
- state.decodeStepCount++;
267
+ advanceDecodeStepCount(state);
1186
268
 
1187
269
  const { hiddenStates, decodeHiddenBuffer, decodeAltBuffer } = await runDecodeLayers(
1188
270
  state,
@@ -1195,51 +277,23 @@ export async function advanceWithTokenAndEmbedding(state, tokenId, opts, helpers
1195
277
  throw new Error('GPU readback disabled; cannot return embedding');
1196
278
  }
1197
279
 
1198
- const device = getDevice();
1199
- if (!device) {
1200
- throw new Error('GPU device not available');
1201
- }
1202
-
1203
280
  const config = state.modelConfig;
1204
281
  const activationDtype = getEffectiveActivationDtype(state, opts);
1205
- const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
1206
282
 
1207
283
  let embedding;
1208
284
  try {
1209
- const sampleSize = config.hiddenSize * activationBytes;
1210
- const staging = device.createBuffer({
1211
- size: sampleSize,
1212
- usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
1213
- });
1214
-
1215
- let decodedHidden;
1216
- let stagingMapped = false;
1217
- try {
1218
- const enc = device.createCommandEncoder({ label: 'advance_with_embedding_copy' });
1219
- enc.copyBufferToBuffer(hiddenStates, 0, staging, 0, sampleSize);
1220
- device.queue.submit([enc.finish()]);
1221
-
1222
- await staging.mapAsync(GPUMapMode.READ);
1223
- stagingMapped = true;
1224
- decodedHidden = decodeReadback(staging.getMappedRange().slice(0), activationDtype);
1225
- } finally {
1226
- if (stagingMapped) {
1227
- staging.unmap();
1228
- }
1229
- staging.destroy();
1230
- }
1231
- const finalNormWeights = await getFinalNormWeights(state);
1232
- embedding = extractEmbeddingFromHidden(
1233
- decodedHidden,
1234
- 1,
1235
- config.hiddenSize,
285
+ const extracted = await extractEmbeddingFromHiddenGPU({
286
+ hiddenBuffer: hiddenStates,
287
+ activationDtype,
288
+ numTokens: 1,
289
+ hiddenSize: config.hiddenSize,
1236
290
  embeddingMode,
1237
- finalNormWeights,
291
+ finalNorm: state.weights.get('final_norm'),
292
+ finalNormBias: state.weights.get('final_norm_bias') ?? null,
1238
293
  config,
1239
- state.embeddingPostprocessor,
1240
- null,
1241
- state.weights.get('final_norm_bias') ?? null
1242
- );
294
+ embeddingPostprocessor: state.embeddingPostprocessor,
295
+ });
296
+ embedding = extracted.embedding;
1243
297
  } finally {
1244
298
  const isPreAllocated = isOwnedDecodeBuffer(hiddenStates, decodeHiddenBuffer, decodeAltBuffer);
1245
299
  if (!isPreAllocated) {
@@ -1337,9 +391,14 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1337
391
 
1338
392
  state.batchingStats.batchedForwardCalls += 1;
1339
393
  const tokensPerInterval = batchSize * readbackInterval;
394
+ const recorderOptions = {
395
+ recordLabels: opts.debug === true || opts.benchmark === true || opts.executionObserver === true,
396
+ recordDispatches: opts.debug === true || opts.executionObserver === true,
397
+ aggregateDispatches: opts.executionObserver === true,
398
+ };
1340
399
  const recorder = opts.profile
1341
- ? createProfilingRecorder('batch_decode', device)
1342
- : createCommandRecorder('batch_decode', { recordLabels: opts.debug === true || opts.benchmark === true }, device);
400
+ ? createProfilingRecorder('batch_decode', device, recorderOptions)
401
+ : createCommandRecorder('batch_decode', recorderOptions, device);
1343
402
  const lmHead = state.weights.get('lm_head');
1344
403
  if (lmHead && isCpuWeightBuffer(lmHead)) {
1345
404
  throw new Error('[Pipeline] GPU-only decode not supported with CPU-resident LM head.');
@@ -1503,19 +562,20 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1503
562
  const embedMetadata = getWeightMetadata(embedBufferRaw);
1504
563
  const activationDtype = getEffectiveActivationDtype(state, opts);
1505
564
 
1506
- // GPU-side repetition penalty: upload deduplicated history before batch
565
+ // Preserve order and duplicates so the window can advance within the batch.
1507
566
  const repetitionPenalty = opts.repetitionPenalty ?? samplingDefaults.repetitionPenalty;
1508
- const repPenaltyWindow = samplingDefaults.repetitionPenaltyWindow;
1509
- if (repetitionPenalty !== 1.0 && currentIds.length > 0) {
1510
- const uniqueTokens = [...new Set(currentIds.slice(-repPenaltyWindow))];
1511
- repHistoryCount = uniqueTokens.length;
1512
- const historyData = new Uint32Array(uniqueTokens);
567
+ const presencePenalty = opts.presencePenalty ?? samplingDefaults.presencePenalty;
568
+ const repPenaltyWindow = opts.repetitionPenaltyWindow ?? samplingDefaults.repetitionPenaltyWindow;
569
+ if (repetitionPenalty !== 1.0 || presencePenalty !== 0) {
570
+ const historyTokens = currentIds.slice(-repPenaltyWindow);
571
+ repHistoryCount = historyTokens.length;
572
+ const historyData = new Uint32Array(historyTokens);
1513
573
  repHistoryBuffer = device.createBuffer({
1514
574
  size: Math.max(4, historyData.byteLength),
1515
575
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1516
576
  label: 'rep_penalty_history',
1517
577
  });
1518
- device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
578
+ if (historyData.byteLength) device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
1519
579
  }
1520
580
 
1521
581
  // Hoist loop-invariant values to avoid repeated rule lookups and allocations.
@@ -1545,7 +605,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1545
605
  hiddenSize: config.hiddenSize,
1546
606
  vocabSize: config.vocabSize,
1547
607
  scaleEmbeddings: config.scaleEmbeddings,
1548
- embeddingScale: config.embeddingScale,
608
+ embeddingScale: config.embeddingScale, embeddingNormalization: config.embeddingNormalization,
1549
609
  recorder,
1550
610
  outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
1551
611
  transpose: state.embeddingTranspose,
@@ -1636,11 +696,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1636
696
  logitsBuffer = logits.logitsBuffer;
1637
697
 
1638
698
  // Apply GPU-side repetition penalty before sampling
1639
- if (repHistoryBuffer && repetitionPenalty !== 1.0) {
699
+ if (repHistoryBuffer) {
1640
700
  await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
1641
701
  vocabSize,
1642
702
  historyCount: repHistoryCount,
1643
703
  penalty: repetitionPenalty,
704
+ presencePenalty,
705
+ repetitionPenaltyWindow: repPenaltyWindow,
1644
706
  batchCount: i,
1645
707
  batchOffset: 1,
1646
708
  logitsDtype,
@@ -1659,18 +721,20 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1659
721
  await recordGPUSample(recorder, logitsBuffer, vocabSize, {
1660
722
  temperature: opts.temperature,
1661
723
  topK: opts.topK,
724
+ topP: opts.topP,
1662
725
  padTokenId,
1663
726
  logitSoftcap,
1664
727
  logitsDtype,
1665
728
  outputBuffer: tokensBuffer,
1666
729
  outputIndex,
1667
730
  greedyThreshold: samplingDefaults.greedyThreshold,
731
+ randomSeed: opts.seed,
1668
732
  });
1669
733
  }
1670
734
  }
1671
735
 
1672
736
  const stopCheck = canUseHotVocabularyBatchDecode
1673
- ? recordCheckHotVocabStop(recorder, {
737
+ ? await recordCheckHotVocabStop(recorder, {
1674
738
  sampledTokenBuffer: tokensBuffer,
1675
739
  nextInputTokenBuffer: pleInputTokensBuffer,
1676
740
  hotTokenIndexMapBuffer: pleHotVocabularyRuntime.hotTokenIndexMapBuffer,
@@ -1682,7 +746,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1682
746
  currentPos,
1683
747
  })
1684
748
  : useGpuStopFlags
1685
- ? recordCheckStop(recorder, {
749
+ ? await recordCheckStop(recorder, {
1686
750
  sampledTokenBuffer: tokensBuffer,
1687
751
  shouldStopBuffer: stopBuffer,
1688
752
  tokenIndex: outputIndex,
@@ -1781,6 +845,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1781
845
  getUniformCache().flushPendingDestruction();
1782
846
 
1783
847
  const tokens = readback.tokens;
848
+ if (tokens.includes(0xFFFFFFFF)) throw new Error('[Sampling] No finite candidate logits.');
1784
849
  const stopFlags = readback.stopFlags;
1785
850
 
1786
851
  if (stopFlags) {