doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -7,27 +7,34 @@ import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selecti
7
7
  import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
8
8
  import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
9
9
  import { allowReadback } from '../../../gpu/perf-guards.js';
10
- import { log, trace } from '../../../debug/index.js';
10
+ import { log, trace, isTraceEnabled } from '../../../debug/index.js';
11
+ import {
12
+ runMatmul,
13
+ runRMSNorm,
14
+ runGeLU,
15
+ runResidualAdd,
16
+ runScale,
17
+ runSoftmax,
18
+ runSoftEmbeddingSplitF16,
19
+ runSoftEmbeddingLogitsF16,
20
+ } from '../../../gpu/kernel-selector.js';
21
+ import { runDiffusionGemmaCanvasStats } from '../../../gpu/kernels/diffusion-gemma-sampling.js';
11
22
  import {
12
23
  CAPTURE_LEVELS,
13
24
  createDefaultCaptureConfig,
14
25
  validateCaptureConfig,
15
26
  } from '../../../debug/index.js';
16
27
  import { validateCallTimeOptions } from '../../../config/param-validator.js';
17
- import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
18
28
  import { selectRuleValue } from '../../../rules/rule-registry.js';
19
- import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
20
- import { f32ToF16Array } from '../../kv-cache/types.js';
21
29
 
22
30
  // Pipeline sub-modules
23
31
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
24
- import { applyChatTemplate, createKVCache, isStopToken } from './init.js';
25
- import { formatChatMessages } from './chat-format.js';
32
+ import { createKVCache, isStopToken } from './init.js';
26
33
  import { embed } from './embed.js';
27
34
  import { processLayer } from './layer.js';
28
- import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
35
+ import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
29
36
  import { OperatorEventEmitter } from './operator-events.js';
30
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
37
+ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata, getLayout } from '../../../gpu/weight-buffer.js';
31
38
  import {
32
39
  decodeStep,
33
40
  decodeStepLogits,
@@ -62,12 +69,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
62
69
  import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
63
70
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
64
71
  import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
72
+ import { drainPendingTsirReads } from './tsir-fixture-writer.js';
65
73
  import {
66
74
  activateFallbackExecutionPlan,
67
75
  hasFallbackExecutionPlan,
68
76
  rebaseExecutionSessionPlan,
69
77
  resetActiveExecutionPlan,
70
78
  resolveMaxBatchDecodeTokens,
79
+ resolvePrefillRecorderChunkLayers,
71
80
  resolveActiveExecutionPlan,
72
81
  setActiveExecutionPlan,
73
82
  } from './execution-plan.js';
@@ -81,538 +90,306 @@ import {
81
90
  preparePerLayerInputs,
82
91
  createPleBufferCache,
83
92
  prefetchPerLayerRow,
84
- ensurePleScaledProjectionNormWeight,
85
- ensurePleGpuHotVocabularyRuntime,
86
- ensurePleGpuSplitTablesRuntime,
87
93
  getPleHotVocabularyRuntime,
88
94
  hasGpuSplitPerLayerInputEmbeddings,
89
95
  hasRangeBackedPerLayerInputEmbeddings,
90
96
  } from './per-layer-inputs.js';
91
97
  import { createTensor } from '../../../gpu/tensor.js';
92
- import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
93
-
94
- let intentBundleModulePromise = null;
95
-
96
- async function getExperimentalIntentBundleModule() {
97
- intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
98
- return intentBundleModulePromise;
99
- }
98
+ import { getQKNormOnesBuffer } from './attention/types.js';
99
+ import {
100
+ getWeightBuffer as getPipelineWeightBuffer,
101
+ getNormWeightBuffer as getPipelineNormWeightBuffer,
102
+ } from './weights.js';
100
103
 
101
- function isStructuredChatRequest(prompt) {
102
- return prompt != null
103
- && typeof prompt === 'object'
104
- && !Array.isArray(prompt)
105
- && Array.isArray(prompt.messages);
104
+ // Extracted standalone helpers
105
+ import {
106
+ resolvePromptInput,
107
+ releasePerLayerInputBuffer,
108
+ normalizePrefixEmbeddingOverride,
109
+ resolvePrefillEmbeddingInputIds,
110
+ resolvePrefillMultimodalBidirectionalSpan,
111
+ applyPrefixEmbeddingOverride,
112
+ resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
113
+ shouldDisablePrefillCommandBatching,
114
+ resolveEffectivePrefillTokenChunkSize,
115
+ } from './generator-prefill-helpers.js';
116
+ import {
117
+ shouldDisableBatchDecodeAfterShortBatch,
118
+ resolveHotVocabularyBatchDecodeAvailability,
119
+ primePleDecodeRuntimeCache,
120
+ recordPrefillProfileStep,
121
+ resolveTokenText,
122
+ usesReplayPrefillDecode,
123
+ assertIncrementalDecodeSupport,
124
+ summarizeExecutionPlan,
125
+ shouldRetryWithFinitenessFallback,
126
+ createUnhandledFinitenessPolicyError,
127
+ resolveTargetPlanKVDtype,
128
+ resolveCurrentKVCacheDtype,
129
+ cloneRuntimeInferenceWithKVDtype,
130
+ } from './generator-decode-policy.js';
131
+
132
+ const SPECIAL_LIKE_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>\n]{1,32}>)$/i;
133
+ const FINITENESS_RESET_WORDS = new Uint32Array(4);
134
+ const tokenizerSuppressionCache = new WeakMap();
135
+ const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
136
+
137
+ export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
138
+ const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
139
+ const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
140
+ const submit = (runtimeSubmit !== undefined && runtimeSubmit !== null)
141
+ ? runtimeSubmit
142
+ : manifestSubmit;
143
+ if (submit === undefined || submit === null) {
144
+ throw new Error('[Pipeline] runtime.inference.session.prefillChunkSubmitMode is required.');
145
+ }
146
+ if (!PREFILL_CHUNK_SUBMIT_MODES.has(submit)) {
147
+ throw new Error(
148
+ `[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
149
+ );
150
+ }
151
+ return submit;
106
152
  }
107
153
 
108
- export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
109
- return hitStop !== true
110
- && Number.isInteger(actualCount)
111
- && Number.isInteger(requestedCount)
112
- && actualCount > 0
113
- && actualCount < requestedCount;
154
+ function getTokenizerSuppressionCache(tokenizer) {
155
+ let cache = tokenizerSuppressionCache.get(tokenizer);
156
+ if (!cache) {
157
+ cache = new Map();
158
+ tokenizerSuppressionCache.set(tokenizer, cache);
159
+ }
160
+ return cache;
114
161
  }
115
162
 
116
- export function resolveHotVocabularyBatchDecodeAvailability({
117
- hasRangeBackedPerLayerInputs,
118
- pleHotVocabularyRuntime,
119
- tokenId,
120
- }) {
121
- if (hasRangeBackedPerLayerInputs !== true) {
122
- return false;
163
+ function decodeSingleTokenForSuppression(tokenizer, tokenId) {
164
+ if (!tokenizer || typeof tokenizer.decode !== 'function') {
165
+ return '';
123
166
  }
124
- if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
125
- return false;
126
- }
127
- const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
128
- if (!Number.isInteger(sentinelIndex)) {
129
- return false;
130
- }
131
- if (!Number.isInteger(tokenId)) {
132
- return false;
167
+ try {
168
+ return tokenizer.decode([tokenId], false, false);
169
+ } catch {
170
+ return '';
133
171
  }
134
- const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
135
- return hotTokenIndex !== sentinelIndex;
136
172
  }
137
173
 
138
- async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
139
- const perLayerInputsSession = resolvePerLayerInputsSession(
140
- state.modelConfig?.perLayerInputsSession ?? null,
141
- state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
142
- );
143
- const materialization = perLayerInputsSession?.materialization;
144
- if (state.debug) {
145
- log.debug(
146
- 'Pipeline',
147
- `PLE materialization session=${materialization ?? 'null'} ` +
148
- `modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
149
- `runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
150
- `manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
151
- );
152
- }
153
- await ensurePleGpuHotVocabularyRuntime({
154
- config: state.modelConfig,
155
- weights: state.weights,
156
- perLayerInputsSession,
157
- debugFlags: state.debugFlags,
158
- tokenizer: state.tokenizer ?? null,
159
- seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
160
- });
161
- await ensurePleGpuSplitTablesRuntime({
162
- config: state.modelConfig,
163
- weights: state.weights,
164
- perLayerInputsSession,
165
- debugFlags: state.debugFlags,
166
- });
167
- await ensurePleScaledProjectionNormWeight({
168
- config: state.modelConfig,
169
- weights: state.weights,
170
- weightConfig: {
171
- rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
172
- },
173
- debugFlags: state.debugFlags,
174
- });
174
+ function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
175
+ if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
176
+ return [];
177
+ }
178
+ if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
179
+ return [];
180
+ }
181
+ const cache = getTokenizerSuppressionCache(tokenizer);
182
+ const cacheKey = [
183
+ vocabSize,
184
+ samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
185
+ samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
186
+ ].join(':');
187
+ const cached = cache.get(cacheKey);
188
+ if (cached) {
189
+ return cached;
190
+ }
191
+ const tokenIds = [];
192
+ for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
193
+ const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
194
+ && typeof tokenizer.isSpecialToken === 'function'
195
+ && tokenizer.isSpecialToken(tokenId);
196
+ const specialLike = samplingConfig.suppressSpecialLikeTokens === true
197
+ && SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
198
+ if (tokenizerSpecial || specialLike) {
199
+ tokenIds.push(tokenId);
200
+ }
201
+ }
202
+ cache.set(cacheKey, tokenIds);
203
+ return tokenIds;
175
204
  }
176
205
 
177
- function recordPrefillProfileStep(state, entry) {
178
- if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
179
- if (!state.stats.prefillProfileSteps) {
180
- state.stats.prefillProfileSteps = [];
206
+ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
207
+ const vocabSize = state.modelConfig?.vocabSize;
208
+ const suppressed = new Set(samplingConfig.suppressTokenIds);
209
+ const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
210
+ const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
211
+ if (Number.isInteger(eosToken)) {
212
+ stopTokenIds.add(eosToken);
181
213
  }
182
- state.stats.prefillProfileSteps.push(entry);
183
- }
184
-
185
- function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
186
- const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
187
- if (typeof prompt === 'string') {
188
- if (useChatTemplate && state.modelConfig.chatTemplateType) {
189
- if (state.modelConfig.chatTemplateType === 'translategemma') {
190
- throw new Error(
191
- `[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
192
- 'Pass { messages: [...] } instead of a plain string prompt.'
193
- );
194
- }
195
- return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
214
+ for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
215
+ if (!stopTokenIds.has(tokenId)) {
216
+ suppressed.add(tokenId);
196
217
  }
197
- return prompt;
198
218
  }
199
-
200
- if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
201
- throw new Error(
202
- `[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
203
- 'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
204
- );
205
- }
206
- const messages = isStructuredChatRequest(prompt)
207
- ? prompt.messages
208
- : (Array.isArray(prompt) ? prompt : null);
209
- if (!messages) {
210
- throw new Error(
211
- `[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
212
- );
213
- }
214
- const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
215
- return formatChatMessages(messages, templateType, chatOptions);
219
+ return [...suppressed];
216
220
  }
217
221
 
218
- function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
219
- if (!buffer) {
222
+ async function traceActivationHealth(label, buffer, dtype, elementCount) {
223
+ if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
220
224
  return;
221
225
  }
222
- const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
223
- if (ownsBuffer) {
224
- return;
225
- }
226
- const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
227
- if (cachedPleBuffer) {
228
- return;
229
- }
230
- if (recorder) {
231
- recorder.trackTemporaryBuffer(buffer);
232
- return;
233
- }
234
- releaseBuffer(buffer);
226
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
227
+ const data = await readBuffer(buffer, elementCount * bytesPerElement);
228
+ trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
235
229
  }
236
230
 
237
- function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
238
- if (override == null) {
239
- return null;
240
- }
241
- if (typeof override !== 'object') {
242
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
243
- }
244
-
245
- const prefixLength = Number(override.prefixLength);
246
- if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
247
- throw new Error(
248
- `[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
249
- );
250
- }
251
- if (prefixLength === 0) {
252
- return null;
253
- }
231
+ function ownsBorrowedWeightBuffer(weight) {
232
+ return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
233
+ }
254
234
 
255
- const offset = Number(override.offset ?? 0);
256
- if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
257
- throw new Error(
258
- `[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
259
- );
235
+ function borrowLinearWeight(weight, label) {
236
+ if (!weight) {
237
+ throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
260
238
  }
261
- if (offset + prefixLength > numTokens) {
239
+ if (isSplitWeightBuffer(weight)) {
262
240
  throw new Error(
263
- `[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
241
+ `DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
264
242
  );
265
243
  }
244
+ return {
245
+ value: getPipelineWeightBuffer(weight, label),
246
+ owned: ownsBorrowedWeightBuffer(weight),
247
+ };
248
+ }
266
249
 
267
- const embeddings = override.embeddings ?? null;
268
- if (!embeddings) {
269
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
250
+ function borrowNormWeight(weight, label) {
251
+ if (!weight) {
252
+ throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
270
253
  }
271
-
272
- const expectedLength = prefixLength * hiddenSize;
273
- if (embeddings instanceof Float32Array) {
274
- if (embeddings.length !== expectedLength) {
275
- throw new Error(
276
- `[Pipeline] ${contextLabel}: embedding override length mismatch ` +
277
- `(expected=${expectedLength}, got=${embeddings.length}).`
278
- );
279
- }
280
- } else if (isGpuBufferInstance(embeddings)) {
281
- const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
282
- if (embeddings.size < expectedBytes) {
283
- throw new Error(
284
- `[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
285
- `(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
286
- );
287
- }
288
- } else {
254
+ if (isSplitWeightBuffer(weight)) {
289
255
  throw new Error(
290
- `[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
256
+ `DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
291
257
  );
292
258
  }
293
-
294
259
  return {
295
- prefixLength,
296
- offset,
297
- embeddings,
298
- expectedLength,
299
- byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
300
- byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
260
+ value: getPipelineNormWeightBuffer(weight, label),
261
+ owned: ownsBorrowedWeightBuffer(weight),
301
262
  };
302
263
  }
303
264
 
304
- function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
305
- if (embeddingInputSpan == null) {
306
- return inputIds;
307
- }
308
- if (typeof embeddingInputSpan !== 'object') {
309
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
265
+ function releaseBorrowedWeight(borrowed) {
266
+ if (!borrowed?.owned) {
267
+ return;
310
268
  }
269
+ const value = borrowed.value;
270
+ releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
271
+ }
311
272
 
312
- const offset = Number(embeddingInputSpan.offset);
313
- const length = Number(embeddingInputSpan.length);
314
- const tokenId = Number(embeddingInputSpan.tokenId);
315
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
316
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
317
- }
318
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
319
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
320
- }
321
- if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
322
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
273
+ function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
274
+ return logitsState != null
275
+ && isWeightBuffer(embeddingWeight)
276
+ && getWeightDtype(embeddingWeight) === 'f16'
277
+ && getLayout(embeddingWeight) === 'row'
278
+ && embeddingTranspose !== true;
279
+ }
280
+
281
+ function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
282
+ const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
283
+ if (value == null) {
284
+ return undefined;
323
285
  }
324
- if (offset + length > inputIds.length) {
286
+ if (!Number.isInteger(value) || value <= 0) {
325
287
  throw new Error(
326
- `[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
327
- `exceeds inputIds length ${inputIds.length}.`
288
+ 'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
328
289
  );
329
290
  }
330
- const replacedInputIds = Array.from(inputIds);
331
- replacedInputIds.fill(tokenId, offset, offset + length);
332
- return replacedInputIds;
291
+ return value;
333
292
  }
334
293
 
335
- function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
336
- if (bidirectionalSpan == null) {
337
- return null;
338
- }
339
- if (typeof bidirectionalSpan !== 'object') {
340
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
294
+ function normalizeCanvasTokenIds(canvas, context) {
295
+ if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
296
+ throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
341
297
  }
298
+ return Array.from(canvas, (value, index) => {
299
+ if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
300
+ throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
301
+ }
302
+ return value;
303
+ });
304
+ }
342
305
 
343
- const offset = Number(bidirectionalSpan.offset);
344
- const length = Number(bidirectionalSpan.length);
345
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
346
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
306
+ function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
307
+ if (logits == null) {
308
+ return null;
347
309
  }
348
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
349
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
310
+ const expected = canvasLength * vocabSize;
311
+ const values = logits instanceof Float32Array
312
+ ? logits
313
+ : (Array.isArray(logits) ? Float32Array.from(logits) : null);
314
+ if (!values) {
315
+ throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
350
316
  }
351
- if ((offset + length) > inputIds.length) {
317
+ if (values.length !== expected) {
352
318
  throw new Error(
353
- `[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
354
- `exceeds inputIds length ${inputIds.length}.`
319
+ `[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
355
320
  );
356
321
  }
357
- return { offset, length };
322
+ return values;
358
323
  }
359
324
 
360
- async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
361
- if (!override) {
362
- return baseTensor;
325
+ function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
326
+ if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
327
+ return null;
328
+ }
329
+ const dtype = logits.logitsDtype;
330
+ if (dtype !== 'f32') {
331
+ throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
363
332
  }
364
- if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
333
+ if (logits.vocabSize !== vocabSize) {
365
334
  throw new Error(
366
- `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
335
+ `[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
367
336
  );
368
337
  }
369
-
370
- const device = getDevice();
371
- if (!device) {
372
- throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
373
- }
374
-
375
- const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
376
- const targetElementOffset = override.offset * hiddenSize;
377
- const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
378
- try {
379
- const encoder = device.createCommandEncoder();
380
- encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
381
- if (baseTensor.dtype === 'f32') {
382
- if (isGpuBufferInstance(override.embeddings)) {
383
- encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
384
- device.queue.submit([encoder.finish()]);
385
- } else {
386
- device.queue.submit([encoder.finish()]);
387
- uploadData(outputBuffer, override.embeddings, override.byteOffset);
388
- }
389
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
390
- }
391
-
392
- if (isGpuBufferInstance(override.embeddings)) {
393
- const overrideTensor = createTensor(
394
- override.embeddings,
395
- 'f32',
396
- [override.prefixLength, hiddenSize],
397
- 'prefill_embedding_override_f32'
398
- );
399
- assertImplicitDtypeTransitionAllowed({
400
- executionPolicies,
401
- fromDtype: 'f32',
402
- toDtype: 'f16',
403
- op: 'prefill_embedding_override',
404
- detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
405
- });
406
- const castedOverride = await castF32ToF16(overrideTensor);
407
- try {
408
- encoder.copyBufferToBuffer(
409
- castedOverride.buffer,
410
- 0,
411
- outputBuffer,
412
- targetByteOffset,
413
- castedOverride.buffer.size
414
- );
415
- device.queue.submit([encoder.finish()]);
416
- } finally {
417
- releaseBuffer(castedOverride.buffer);
418
- }
419
- } else {
420
- assertImplicitDtypeTransitionAllowed({
421
- executionPolicies,
422
- fromDtype: 'f32',
423
- toDtype: 'f16',
424
- op: 'prefill_embedding_override',
425
- detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
426
- });
427
- const packedOverride = f32ToF16Array(override.embeddings);
428
- device.queue.submit([encoder.finish()]);
429
- uploadData(outputBuffer, packedOverride, targetByteOffset);
430
- }
431
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
432
- } catch (error) {
433
- releaseBuffer(outputBuffer);
434
- throw error;
435
- }
436
- }
437
-
438
- /**
439
- * Resolve display text for a token ID sequence.
440
- *
441
- * Preference order:
442
- * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
443
- * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
444
- * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
445
- * unless primary returned empty string and fallback looks like a special token (<...>),
446
- * in which case empty string is preserved to keep skip-special behavior deterministic.
447
- * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
448
- * 5. fallbackText — static fallback string (default '?').
449
- */
450
- function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
451
- const renderPrimary = typeof renderTokenText === 'function'
452
- ? renderTokenText
453
- : (ids) => tokenizer?.decode?.(ids);
454
- const renderFallback = typeof renderFallbackTokenText === 'function'
455
- ? renderFallbackTokenText
456
- : (ids) => tokenizer?.decode?.(ids, false);
457
-
458
- const primaryText = renderPrimary(tokenIds);
459
- if (typeof primaryText === 'string' && primaryText.length > 0) {
460
- return primaryText;
461
- }
462
-
463
- const fallback = renderFallback(tokenIds);
464
- if (typeof fallback === 'string' && fallback.length > 0) {
465
- // Keep skip-special behavior deterministic: if primary decoding filtered this
466
- // token to empty, do not reintroduce obvious special-token text via fallback.
467
- if (
468
- primaryText === ''
469
- && /^<[^>\n]{1,80}>$/.test(fallback.trim())
470
- ) {
471
- return '';
472
- }
473
- return fallback;
474
- }
475
-
476
- return fallbackText;
477
- }
478
-
479
- function usesReplayPrefillDecode(state) {
480
- return state?.modelConfig?.decodeStrategy === 'replay_prefill';
481
- }
482
-
483
- export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
484
- if (
485
- opts?.disableCommandBatching === true
486
- || opts?.debug === true
487
- || (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
488
- ) {
489
- return true;
490
- }
491
- if (state?.kvCache?.layout === 'bdpa_paged') {
492
- return true;
493
- }
494
- if (multimodalBidirectionalSpan == null) {
495
- return false;
496
- }
497
- if (state?.kvCache?.hasGPUCache?.() !== true) {
498
- return false;
338
+ if (logits.canvasLength !== canvasLength) {
339
+ throw new Error(
340
+ `[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
341
+ );
499
342
  }
500
- // WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
501
- // multimodal logits on the first generated token. Keep this lane fail-closed
502
- // until the recorder/cache interaction is numerically matched to the direct path.
503
- return true;
504
- }
505
-
506
- function assertIncrementalDecodeSupport(state, operation) {
507
- if (!usesReplayPrefillDecode(state)) {
508
- return;
343
+ const temperature = logits.temperature ?? 1.0;
344
+ if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
345
+ throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
509
346
  }
510
- throw new Error(
511
- `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
512
- 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
513
- 'explicit layerTypes for mixed-geometry/shared-KV decode.'
514
- );
347
+ return {
348
+ logitsBuffer: logits.logitsBuffer,
349
+ logitsDtype: dtype,
350
+ temperature,
351
+ releaseOnUse: logits.releaseOnUse !== false,
352
+ };
515
353
  }
516
354
 
517
- function summarizeExecutionPlan(plan) {
518
- if (!plan) {
355
+ function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
356
+ if (state?.kind !== 'soft_embedding') {
519
357
  return null;
520
358
  }
521
- if (typeof plan !== 'object') {
522
- log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
523
- return null;
359
+ if (!isGpuBufferInstance(state.buffer)) {
360
+ throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
524
361
  }
525
- if (typeof plan.id !== 'string') {
526
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
362
+ const dtype = state.dtype ?? 'f32';
363
+ if (dtype !== 'f32') {
364
+ throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
527
365
  }
528
- if (typeof plan.activationDtype !== 'string') {
529
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
366
+ if (state.canvasLength !== canvasLength) {
367
+ throw new Error(
368
+ `[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
369
+ `expected ${canvasLength}, got ${state.canvasLength}.`
370
+ );
371
+ }
372
+ if (state.hiddenSize !== hiddenSize) {
373
+ throw new Error(
374
+ `[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
375
+ `expected ${hiddenSize}, got ${state.hiddenSize}.`
376
+ );
530
377
  }
531
378
  return {
532
- id: plan.id,
533
- kernelPathId: plan.kernelPathId ?? null,
534
- kernelPathSource: plan.kernelPathSource ?? 'none',
535
- activationDtype: plan.activationDtype,
536
- readbackInterval: plan.readbackInterval ?? null,
537
- batchSize: plan.defaultBatchSize,
538
- stopCheckMode: plan.defaultStopCheckMode,
539
- ringTokens: plan.ringTokens ?? null,
540
- ringStop: plan.ringStop ?? null,
541
- ringStaging: plan.ringStaging ?? null,
379
+ buffer: state.buffer,
380
+ dtype,
381
+ releaseOnUse: state.releaseOnUse !== false,
382
+ scaled: state.scaled === true,
542
383
  };
543
384
  }
544
385
 
545
- export function shouldRetryWithFinitenessFallback(error) {
546
- if (error?.name === 'FinitenessError') {
547
- return true;
548
- }
549
- const message = typeof error?.message === 'string'
550
- ? error.message
551
- : (typeof error === 'string' ? error : '');
552
- if (!message.startsWith('[Sampling]')) {
553
- return false;
554
- }
555
- return message.includes('no finite candidate logits after masking the pad token')
556
- || message.includes('Softmax produced no finite candidate probabilities');
557
- }
558
-
559
- function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
560
- const activePlan = resolveActiveExecutionPlan(state);
561
- const wrapped = new Error(
562
- `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
563
- `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
564
- 'Resolve the unstable path with an explicit capability-aware execution override, ' +
565
- 'or opt into alternate-plan recovery with ' +
566
- 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
567
- error instanceof Error ? { cause: error } : undefined
568
- );
569
- wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
570
- return wrapped;
571
- }
572
-
573
- function assertResolvedKVDtype(kvDtype, contextLabel) {
574
- if (kvDtype === 'f16' || kvDtype === 'f32') {
575
- return kvDtype;
576
- }
577
- throw new Error(
578
- `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
579
- `got ${JSON.stringify(kvDtype)}.`
580
- );
581
- }
582
-
583
- function resolveTargetPlanKVDtype(plan, contextLabel) {
584
- return assertResolvedKVDtype(
585
- plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
586
- contextLabel
587
- );
588
- }
386
+ let intentBundleModulePromise = null;
589
387
 
590
- function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
591
- return assertResolvedKVDtype(
592
- state?.kvCache?.kvDtype
593
- ?? plan?.kernelPath?.kvDtype
594
- ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
595
- ?? plan?.activationDtype
596
- ?? null,
597
- contextLabel
598
- );
388
+ async function getExperimentalIntentBundleModule() {
389
+ intentBundleModulePromise ??= import('../../../experimental/hotswap/intent-bundle.js');
390
+ return intentBundleModulePromise;
599
391
  }
600
392
 
601
- function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
602
- const runtimeInference = state?.runtimeConfig?.inference;
603
- if (!runtimeInference?.session?.kvcache) {
604
- throw new Error(
605
- '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
606
- );
607
- }
608
- return mergeRuntimeValues(runtimeInference, {
609
- session: {
610
- kvcache: {
611
- kvDtype,
612
- },
613
- },
614
- });
615
- }
616
393
 
617
394
  export class PipelineGenerator {
618
395
 
@@ -751,8 +528,14 @@ export class PipelineGenerator {
751
528
  this.#state.stats.ttftMs = 0;
752
529
  this.#state.stats.decodeTimeMs = 0;
753
530
  this.#state.stats.decodeRecordMs = 0;
531
+ this.#state.stats.decodeRecordOps = 0;
532
+ this.#state.stats.decodeRecordPasses = 0;
533
+ this.#state.stats.decodeRecordOpLabels = {};
754
534
  this.#state.stats.decodeSubmitWaitMs = 0;
755
535
  this.#state.stats.decodeReadbackWaitMs = 0;
536
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
537
+ this.#state.stats.decodeReadbackCleanupMs = 0;
538
+ this.#state.stats.decodeReadbackCopyMs = 0;
756
539
  this.#state.decodeStepCount = 0;
757
540
  this.#state.disableRecordedLogits = false;
758
541
  this.#state.disableFusedDecode = false;
@@ -762,6 +545,12 @@ export class PipelineGenerator {
762
545
  totalBatchedTimeMs: 0,
763
546
  totalUnbatchedTimeMs: 0,
764
547
  gpuSubmissions: 0,
548
+ requestedBatchTokens: 0,
549
+ effectiveBatchTokens: 0,
550
+ executedBatchTokens: 0,
551
+ resolvedBatchTokens: 0,
552
+ maxBatchTokenCap: null,
553
+ batchClampCount: 0,
765
554
  };
766
555
  resetActiveExecutionPlan(this.#state);
767
556
  this.#state.decodeRing?.reset();
@@ -812,14 +601,96 @@ export class PipelineGenerator {
812
601
  _sampleNextTokenFromLogits(logits, generatedIds, opts) {
813
602
  const sampledLogits = Float32Array.from(logits);
814
603
  applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
604
+ // Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
605
+ // implement grammar/schema-constrained decoding. The hook receives the
606
+ // mutable logit buffer (after repetition penalty) plus the running token
607
+ // sequence so it can track parse state across decode steps.
608
+ if (typeof opts?.logitMaskFn === "function") {
609
+ try {
610
+ opts.logitMaskFn(sampledLogits, {
611
+ generatedIds,
612
+ tokenizer: this.#state.tokenizer ?? null,
613
+ vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
614
+ });
615
+ } catch (maskError) {
616
+ log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
617
+ }
618
+ }
815
619
  const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
816
- return sample(sampledLogits, {
620
+ const tokenId = sample(sampledLogits, {
817
621
  temperature: opts.temperature,
818
622
  topP: opts.topP,
819
623
  topK: opts.topK,
820
624
  padTokenId,
821
625
  seed: opts.seed,
626
+ suppressTokenIds: opts.suppressTokenIds,
822
627
  });
628
+ if (typeof opts.onLogits === 'function') {
629
+ opts.onLogits(sampledLogits, {
630
+ tokenId,
631
+ inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
632
+ });
633
+ }
634
+ return tokenId;
635
+ }
636
+
637
+ _resolvePrefillTokenChunkSize(inputIds) {
638
+ const chunkSize = resolveEffectivePrefillTokenChunkSize(this.#state);
639
+ if (chunkSize === undefined) {
640
+ throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
641
+ }
642
+ if (chunkSize === null) {
643
+ return null;
644
+ }
645
+ if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
646
+ throw new Error('inference.session.prefillTokenChunkSize must be null or a positive integer.');
647
+ }
648
+ return chunkSize < inputIds.length ? chunkSize : null;
649
+ }
650
+
651
+ async _commitPrefillHiddenChunk(prefillResult) {
652
+ const {
653
+ numTokens,
654
+ startPos,
655
+ currentRecorder,
656
+ recordProfile,
657
+ currentHiddenBuffer,
658
+ } = prefillResult;
659
+
660
+ try {
661
+ if (currentRecorder) {
662
+ await currentRecorder.submitAndWait();
663
+ await recordProfile(currentRecorder);
664
+ } else {
665
+ const device = getDevice();
666
+ if (device) {
667
+ await device.queue.onSubmittedWorkDone();
668
+ }
669
+ }
670
+ this.#state.currentSeqLen = startPos + numTokens;
671
+ } finally {
672
+ releaseBuffer(currentHiddenBuffer);
673
+ }
674
+ }
675
+
676
+ async _prefillInputIdsToLogits(inputIds, opts) {
677
+ const chunkSize = this._resolvePrefillTokenChunkSize(inputIds);
678
+ if (chunkSize === null) {
679
+ return this._prefill(inputIds, opts);
680
+ }
681
+
682
+ for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
683
+ const end = Math.min(offset + chunkSize, inputIds.length);
684
+ const chunk = inputIds.slice(offset, end);
685
+ const isFinalChunk = end === inputIds.length;
686
+ if (isFinalChunk) {
687
+ return this._prefill(chunk, opts);
688
+ }
689
+ const prefillResult = await this._prefillToHidden(chunk, opts);
690
+ await this._commitPrefillHiddenChunk(prefillResult);
691
+ }
692
+
693
+ return this._prefill(inputIds, opts);
823
694
  }
824
695
 
825
696
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
@@ -830,8 +701,9 @@ export class PipelineGenerator {
830
701
  }
831
702
 
832
703
  let logits;
704
+ const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
833
705
  try {
834
- logits = await this._prefill(inputIds, opts);
706
+ logits = await runPrefill();
835
707
  } catch (error) {
836
708
  if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
837
709
  throw error;
@@ -841,7 +713,7 @@ export class PipelineGenerator {
841
713
  opts,
842
714
  contextLabel,
843
715
  opts.maxTokens ?? 1,
844
- () => this._prefill(inputIds, opts),
716
+ runPrefill,
845
717
  prefillStartSeqLen
846
718
  );
847
719
  }
@@ -880,6 +752,11 @@ export class PipelineGenerator {
880
752
  return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
881
753
  }
882
754
 
755
+ _recordStopReason(reason, tokenId = null) {
756
+ this.#state.stats.stopReason = reason;
757
+ this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
758
+ }
759
+
883
760
  async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
884
761
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
885
762
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
@@ -891,10 +768,21 @@ export class PipelineGenerator {
891
768
  this.#state.stats.gpuTimePrefillMs = undefined;
892
769
  this.#state.stats.gpuTimeDecodeMs = undefined;
893
770
  this.#state.stats.decodeRecordMs = 0;
771
+ this.#state.stats.decodeRecordOps = 0;
772
+ this.#state.stats.decodeRecordPasses = 0;
773
+ this.#state.stats.decodeRecordOpLabels = {};
894
774
  this.#state.stats.decodeSubmitWaitMs = 0;
895
775
  this.#state.stats.decodeReadbackWaitMs = 0;
776
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
777
+ this.#state.stats.decodeReadbackCleanupMs = 0;
778
+ this.#state.stats.decodeReadbackCopyMs = 0;
779
+ this.#state.stats.prefillRecordMs = 0;
780
+ this.#state.stats.prefillSubmitWaitMs = 0;
896
781
  this.#state.stats.singleTokenSubmitWaitMs = 0;
897
782
  this.#state.stats.singleTokenReadbackWaitMs = 0;
783
+ this.#state.stats.singleTokenReadbackMapWaitMs = 0;
784
+ this.#state.stats.singleTokenReadbackCleanupMs = 0;
785
+ this.#state.stats.singleTokenReadbackCopyMs = 0;
898
786
  this.#state.stats.singleTokenOrchestrationMs = 0;
899
787
  this.#state.stats.pleHotVocabularyHits = 0;
900
788
  this.#state.stats.pleHotVocabularyMisses = 0;
@@ -904,19 +792,25 @@ export class PipelineGenerator {
904
792
  this.#state.stats.plePreparedTokenCacheBytes = 0;
905
793
  this.#state.stats.decodeMode = null;
906
794
  this.#state.stats.batchGuardReason = null;
795
+ this.#state.stats.stopReason = null;
796
+ this.#state.stats.stopTokenId = null;
907
797
  this.#state.stats.ttftMs = 0;
908
798
  const startTime = performance.now();
909
799
 
910
800
  const opts = resolveGenerateOptions(this.#state, options);
801
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
802
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
911
803
  // Validate and normalize sampling parameters through single source of truth
912
804
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
913
805
  opts.temperature = samplingConfig.temperature;
914
806
  opts.topP = samplingConfig.topP;
915
807
  opts.topK = samplingConfig.topK;
916
808
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
809
+ opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
917
810
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
918
811
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
919
- if (diagnosticsEnabled) {
812
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
813
+ if (diagnosticsEnabled || tsirFixtureCfg) {
920
814
  const captureConfig = {
921
815
  ...createDefaultCaptureConfig(),
922
816
  enabled: true,
@@ -925,13 +819,20 @@ export class PipelineGenerator {
925
819
  };
926
820
  validateCaptureConfig(captureConfig);
927
821
  this.#state.operatorDiagnostics = {
928
- enabled: true,
822
+ enabled: diagnosticsEnabled === true,
929
823
  captureConfig,
930
- emitter: new OperatorEventEmitter({
824
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
931
825
  modelHash: this.#state.manifest?.modelId ?? null,
932
826
  runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
933
827
  executionPlanHash: opts.executionPlan?.id ?? null,
934
- }),
828
+ }) : null,
829
+ tsirFixture: tsirFixtureCfg ? {
830
+ dir: tsirFixtureCfg.dir,
831
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
832
+ ? tsirFixtureCfg.layerFilter
833
+ : null,
834
+ records: [],
835
+ } : null,
935
836
  };
936
837
  }
937
838
  const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
@@ -1051,6 +952,7 @@ export class PipelineGenerator {
1051
952
  yield* emitToken(this, firstToken, decodeToken);
1052
953
 
1053
954
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
955
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1054
956
  this.#state.stats.decodeTimeMs = 0;
1055
957
  this.#state.stats.tokensGenerated = 1;
1056
958
  this.#state.stats.decodeTokens = 1;
@@ -1277,6 +1179,599 @@ export class PipelineGenerator {
1277
1179
  // Generation Public API
1278
1180
  // ==========================================================================
1279
1181
 
1182
+ /*
1183
+ * Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
1184
+ * match. Intended for "prefix-reuse" workflows where a caller wants to run
1185
+ * several decodes that share a common prompt prefix: prefill once with the
1186
+ * shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
1187
+ * drop the tail's KV entries and reuse the prefix KV for the next tail.
1188
+ *
1189
+ * Only valid when no decode is in progress.
1190
+ */
1191
+ resetToSeqLen(seqLen) {
1192
+ if (this.#state.isGenerating) {
1193
+ throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
1194
+ }
1195
+ const target = Math.max(0, Math.floor(Number(seqLen) || 0));
1196
+ if (!Number.isFinite(target)) {
1197
+ throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
1198
+ }
1199
+ if (target > this.#state.currentSeqLen) {
1200
+ throw new Error(
1201
+ `InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
1202
+ );
1203
+ }
1204
+ this.#state.kvCache?.truncate?.(target);
1205
+ this.#state.currentSeqLen = target;
1206
+ }
1207
+
1208
+ async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
1209
+ const config = this.#state.modelConfig;
1210
+ const canvasLength = canvasIds.length;
1211
+ const hiddenSize = config.hiddenSize;
1212
+ const vocabSize = config.vocabSize;
1213
+ const elementCount = canvasLength * hiddenSize;
1214
+ const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
1215
+ selfConditioningLogits,
1216
+ canvasLength,
1217
+ hiddenSize
1218
+ );
1219
+ const logitsState = softEmbeddingState
1220
+ ? null
1221
+ : normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
1222
+ const logits = (softEmbeddingState || logitsState)
1223
+ ? null
1224
+ : normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
1225
+ const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
1226
+ if (!weights || typeof weights !== 'object') {
1227
+ throw new Error(
1228
+ 'DiffusionGemma self-conditioning weights were not loaded. ' +
1229
+ 'Expected model.decoder.self_conditioning tensors in the manifest.'
1230
+ );
1231
+ }
1232
+
1233
+ const embedBufferRaw = this.#state.weights.get('embed');
1234
+ if (!embedBufferRaw) {
1235
+ throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
1236
+ }
1237
+ const borrowed = {
1238
+ preNorm: null,
1239
+ gateProj: null,
1240
+ upProj: null,
1241
+ downProj: null,
1242
+ softEmbedding: null,
1243
+ };
1244
+ let softLogitsTensor = null;
1245
+ let softLogitsOwned = true;
1246
+ let softEmbeddingStateTensor = null;
1247
+ let softEmbeddingStateOwned = false;
1248
+ let softmaxTensor = null;
1249
+ let softEmbeddings = null;
1250
+ let scaledSoftEmbeddings = null;
1251
+ let baseEmbeddings = null;
1252
+ let normedSoft = null;
1253
+ let gate = null;
1254
+ let up = null;
1255
+ let activated = null;
1256
+ let down = null;
1257
+ let combined = null;
1258
+ let output = null;
1259
+
1260
+ const releaseTensorOnce = (() => {
1261
+ const released = new Set();
1262
+ return (tensor) => {
1263
+ const buffer = tensor?.buffer ?? null;
1264
+ if (!buffer || released.has(buffer)) return;
1265
+ released.add(buffer);
1266
+ releaseBuffer(buffer);
1267
+ };
1268
+ })();
1269
+
1270
+ try {
1271
+ borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
1272
+ borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
1273
+ borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
1274
+ borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
1275
+ if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
1276
+ throw new Error(
1277
+ 'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
1278
+ );
1279
+ }
1280
+ borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
1281
+ ? null
1282
+ : borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1283
+
1284
+ const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1285
+ const embedDtype = isCpuWeightBuffer(embedBufferRaw)
1286
+ ? embedBufferRaw.dtype
1287
+ : getWeightDtype(embedBufferRaw);
1288
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
1289
+ baseEmbeddings = await embed(canvasIds, embedBuffer, {
1290
+ hiddenSize,
1291
+ vocabSize,
1292
+ scaleEmbeddings: config.scaleEmbeddings,
1293
+ debug: opts.debug,
1294
+ recorder: null,
1295
+ transpose: this.#state.embeddingTranspose,
1296
+ activationDtype: 'f32',
1297
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
1298
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1299
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1300
+ operatorDiagnostics: this.#state.operatorDiagnostics,
1301
+ });
1302
+
1303
+ if (softEmbeddingState) {
1304
+ softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
1305
+ softEmbeddings = createTensor(
1306
+ softEmbeddingState.buffer,
1307
+ softEmbeddingState.dtype,
1308
+ [canvasLength, hiddenSize],
1309
+ 'diffusion_gemma_self_conditioning_soft_embedding'
1310
+ );
1311
+ softEmbeddingStateTensor = softEmbeddings;
1312
+ if (!softEmbeddingState.scaled) {
1313
+ scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1314
+ count: elementCount,
1315
+ });
1316
+ if (softEmbeddingStateOwned) {
1317
+ releaseTensorOnce(softEmbeddings);
1318
+ }
1319
+ softEmbeddings = scaledSoftEmbeddings;
1320
+ softEmbeddingStateTensor = null;
1321
+ scaledSoftEmbeddings = null;
1322
+ }
1323
+ } else if (logitsState || logits) {
1324
+ const softmaxTemperature = logitsState?.temperature ?? 1.0;
1325
+ if (logitsState) {
1326
+ softLogitsOwned = logitsState.releaseOnUse;
1327
+ softLogitsTensor = createTensor(
1328
+ logitsState.logitsBuffer,
1329
+ logitsState.logitsDtype,
1330
+ [canvasLength, vocabSize],
1331
+ 'diffusion_gemma_self_conditioning_logits'
1332
+ );
1333
+ } else {
1334
+ const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
1335
+ uploadData(logitsBuffer, logits);
1336
+ softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
1337
+ }
1338
+ if (canUseChunkedSoftEmbeddingLogits(
1339
+ logitsState,
1340
+ borrowed.softEmbedding?.value,
1341
+ this.#state.embeddingTranspose
1342
+ )) {
1343
+ softEmbeddings = await runSoftEmbeddingLogitsF16(
1344
+ softLogitsTensor,
1345
+ borrowed.softEmbedding.value,
1346
+ canvasLength,
1347
+ hiddenSize,
1348
+ vocabSize,
1349
+ {
1350
+ temperature: softmaxTemperature,
1351
+ chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1352
+ }
1353
+ );
1354
+ } else {
1355
+ softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
1356
+ batchSize: canvasLength,
1357
+ size: vocabSize,
1358
+ temperature: softmaxTemperature,
1359
+ });
1360
+ softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
1361
+ ? await runSoftEmbeddingSplitF16(
1362
+ softmaxTensor,
1363
+ embedBufferRaw,
1364
+ canvasLength,
1365
+ hiddenSize,
1366
+ vocabSize
1367
+ )
1368
+ : await runMatmul(
1369
+ softmaxTensor,
1370
+ borrowed.softEmbedding.value,
1371
+ canvasLength,
1372
+ hiddenSize,
1373
+ vocabSize,
1374
+ {
1375
+ transposeB: this.#state.embeddingTranspose === true,
1376
+ role: 'diffusion_gemma_self_conditioning_embed',
1377
+ outputDtype: 'f32',
1378
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1379
+ }
1380
+ );
1381
+ }
1382
+ scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1383
+ count: elementCount,
1384
+ });
1385
+ if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1386
+ releaseTensorOnce(softEmbeddings);
1387
+ }
1388
+ softEmbeddings = scaledSoftEmbeddings;
1389
+ scaledSoftEmbeddings = null;
1390
+ } else {
1391
+ const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
1392
+ uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
1393
+ softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
1394
+ }
1395
+
1396
+ normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
1397
+ batchSize: canvasLength,
1398
+ hiddenSize,
1399
+ rmsNormWeightOffset: false,
1400
+ });
1401
+ const intermediateSize = config.intermediateSize;
1402
+ gate = await runMatmul(
1403
+ normedSoft,
1404
+ borrowed.gateProj.value,
1405
+ canvasLength,
1406
+ intermediateSize,
1407
+ hiddenSize,
1408
+ {
1409
+ transposeB: 'auto',
1410
+ role: 'diffusion_gemma_self_conditioning_gate',
1411
+ outputDtype: 'f32',
1412
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1413
+ }
1414
+ );
1415
+ up = await runMatmul(
1416
+ normedSoft,
1417
+ borrowed.upProj.value,
1418
+ canvasLength,
1419
+ intermediateSize,
1420
+ hiddenSize,
1421
+ {
1422
+ transposeB: 'auto',
1423
+ role: 'diffusion_gemma_self_conditioning_up',
1424
+ outputDtype: 'f32',
1425
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1426
+ }
1427
+ );
1428
+ activated = await runGeLU(up, {
1429
+ size: canvasLength * intermediateSize,
1430
+ gate,
1431
+ });
1432
+ down = await runMatmul(
1433
+ activated,
1434
+ borrowed.downProj.value,
1435
+ canvasLength,
1436
+ hiddenSize,
1437
+ intermediateSize,
1438
+ {
1439
+ transposeB: 'auto',
1440
+ role: 'diffusion_gemma_self_conditioning_down',
1441
+ outputDtype: 'f32',
1442
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1443
+ }
1444
+ );
1445
+ combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
1446
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1447
+ });
1448
+ const postNormWeight = weights.postNorm
1449
+ ? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
1450
+ : {
1451
+ value: getQKNormOnesBuffer(hiddenSize),
1452
+ owned: false,
1453
+ };
1454
+ borrowed.postNorm = postNormWeight;
1455
+ output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
1456
+ batchSize: canvasLength,
1457
+ hiddenSize,
1458
+ rmsNormWeightOffset: false,
1459
+ });
1460
+ return output;
1461
+ } catch (error) {
1462
+ releaseTensorOnce(output);
1463
+ throw error;
1464
+ } finally {
1465
+ releaseTensorOnce(combined);
1466
+ releaseTensorOnce(down);
1467
+ releaseTensorOnce(activated);
1468
+ releaseTensorOnce(up);
1469
+ releaseTensorOnce(gate);
1470
+ releaseTensorOnce(normedSoft);
1471
+ if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1472
+ releaseTensorOnce(softEmbeddings);
1473
+ }
1474
+ releaseTensorOnce(softmaxTensor);
1475
+ if (softLogitsOwned) {
1476
+ releaseTensorOnce(softLogitsTensor);
1477
+ }
1478
+ releaseTensorOnce(baseEmbeddings);
1479
+ releaseBorrowedWeight(borrowed.postNorm);
1480
+ releaseBorrowedWeight(borrowed.downProj);
1481
+ releaseBorrowedWeight(borrowed.upProj);
1482
+ releaseBorrowedWeight(borrowed.gateProj);
1483
+ releaseBorrowedWeight(borrowed.preNorm);
1484
+ releaseBorrowedWeight(borrowed.softEmbedding);
1485
+ }
1486
+ }
1487
+
1488
+ async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
1489
+ const embedBufferRaw = this.#state.weights.get('embed');
1490
+ if (!canUseChunkedSoftEmbeddingLogits(
1491
+ logitsState,
1492
+ embedBufferRaw,
1493
+ this.#state.embeddingTranspose
1494
+ )) {
1495
+ return null;
1496
+ }
1497
+
1498
+ const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1499
+ let softEmbedding = null;
1500
+ let scaledSoftEmbedding = null;
1501
+ try {
1502
+ const logitsTensor = createTensor(
1503
+ logitsState.logitsBuffer,
1504
+ logitsState.logitsDtype,
1505
+ [canvasLength, vocabSize],
1506
+ 'diffusion_gemma_self_conditioning_logits'
1507
+ );
1508
+ softEmbedding = await runSoftEmbeddingLogitsF16(
1509
+ logitsTensor,
1510
+ borrowed.value,
1511
+ canvasLength,
1512
+ hiddenSize,
1513
+ vocabSize,
1514
+ {
1515
+ temperature: logitsState.temperature,
1516
+ chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1517
+ }
1518
+ );
1519
+ scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
1520
+ count: canvasLength * hiddenSize,
1521
+ });
1522
+ const returnedBuffer = scaledSoftEmbedding.buffer;
1523
+ return {
1524
+ kind: 'soft_embedding',
1525
+ buffer: returnedBuffer,
1526
+ dtype: scaledSoftEmbedding.dtype,
1527
+ canvasLength,
1528
+ hiddenSize,
1529
+ scaled: true,
1530
+ releaseOnUse: true,
1531
+ release() {
1532
+ releaseBuffer(returnedBuffer);
1533
+ },
1534
+ };
1535
+ } finally {
1536
+ if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
1537
+ releaseBuffer(softEmbedding.buffer);
1538
+ }
1539
+ releaseBorrowedWeight(borrowed);
1540
+ }
1541
+ }
1542
+
1543
+ async computeDiffusionGemmaCanvasLogits(args, options = {}) {
1544
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
1545
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
1546
+ throw new Error('Generation already in progress');
1547
+ }
1548
+ const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
1549
+ if (canvasIds.length === 0) {
1550
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
1551
+ }
1552
+ this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
1553
+ const seqLenBefore = this.#state.currentSeqLen;
1554
+ const opts = {
1555
+ ...resolvePrefillOptions(this.#state, {
1556
+ ...options,
1557
+ useChatTemplate: false,
1558
+ }),
1559
+ _diffusionGemmaDecoder: true,
1560
+ };
1561
+ let selfConditioned = null;
1562
+ let currentHiddenBuffer = null;
1563
+ try {
1564
+ selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1565
+ canvasIds,
1566
+ args?.selfConditioningLogits ?? null,
1567
+ opts
1568
+ );
1569
+ const prefillResult = await this._prefillToHidden(canvasIds, {
1570
+ ...opts,
1571
+ embeddingOverrides: {
1572
+ offset: 0,
1573
+ prefixLength: canvasIds.length,
1574
+ embeddings: selfConditioned.buffer,
1575
+ },
1576
+ });
1577
+ const {
1578
+ numTokens,
1579
+ currentRecorder,
1580
+ recordProfile,
1581
+ debugCheckBuffer,
1582
+ } = prefillResult;
1583
+ currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1584
+ if (currentRecorder) {
1585
+ await currentRecorder.submitAndWait();
1586
+ await recordProfile(currentRecorder);
1587
+ } else {
1588
+ const device = getDevice();
1589
+ if (device) {
1590
+ await device.queue.onSubmittedWorkDone();
1591
+ }
1592
+ }
1593
+ const logits = await computeLogits(
1594
+ currentHiddenBuffer,
1595
+ numTokens,
1596
+ getLogitsWeights(this.#state),
1597
+ getLogitsConfig(this.#state),
1598
+ this.#state.useGPU,
1599
+ this.#state.debugFlags,
1600
+ undefined,
1601
+ debugCheckBuffer,
1602
+ this.#state.runtimeConfig.shared.debug.probes,
1603
+ { lastPositionOnly: false },
1604
+ this.#state.operatorDiagnostics
1605
+ );
1606
+ const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
1607
+ if (logits.length !== expected) {
1608
+ throw new Error(
1609
+ `[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
1610
+ );
1611
+ }
1612
+ return logits;
1613
+ } finally {
1614
+ this.#state.currentSeqLen = seqLenBefore;
1615
+ if (currentHiddenBuffer) {
1616
+ releaseBuffer(currentHiddenBuffer);
1617
+ }
1618
+ if (selfConditioned?.buffer) {
1619
+ releaseBuffer(selfConditioned.buffer);
1620
+ }
1621
+ }
1622
+ }
1623
+
1624
+ async computeDiffusionGemmaCanvasStep(args, options = {}) {
1625
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
1626
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
1627
+ throw new Error('Generation already in progress');
1628
+ }
1629
+ const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
1630
+ if (canvasIds.length === 0) {
1631
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
1632
+ }
1633
+ this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
1634
+ const temperature = args?.temperature;
1635
+ if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
1636
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
1637
+ }
1638
+ const seqLenBefore = this.#state.currentSeqLen;
1639
+ const opts = {
1640
+ ...resolvePrefillOptions(this.#state, {
1641
+ ...options,
1642
+ useChatTemplate: false,
1643
+ }),
1644
+ _diffusionGemmaDecoder: true,
1645
+ };
1646
+ let selfConditioned = null;
1647
+ let currentHiddenBuffer = null;
1648
+ let logitsBuffer = null;
1649
+ let statsBuffers = null;
1650
+ try {
1651
+ selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1652
+ canvasIds,
1653
+ args?.selfConditioningLogits ?? null,
1654
+ opts
1655
+ );
1656
+ const prefillResult = await this._prefillToHidden(canvasIds, {
1657
+ ...opts,
1658
+ embeddingOverrides: {
1659
+ offset: 0,
1660
+ prefixLength: canvasIds.length,
1661
+ embeddings: selfConditioned.buffer,
1662
+ },
1663
+ });
1664
+ const {
1665
+ numTokens,
1666
+ currentRecorder,
1667
+ recordProfile,
1668
+ debugCheckBuffer,
1669
+ } = prefillResult;
1670
+ currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1671
+ if (currentRecorder) {
1672
+ await currentRecorder.submitAndWait();
1673
+ await recordProfile(currentRecorder);
1674
+ } else {
1675
+ const device = getDevice();
1676
+ if (device) {
1677
+ await device.queue.onSubmittedWorkDone();
1678
+ }
1679
+ }
1680
+ if (selfConditioned?.buffer) {
1681
+ releaseBuffer(selfConditioned.buffer);
1682
+ selfConditioned = null;
1683
+ }
1684
+
1685
+ const logitsResult = await computeLogitsGPU(
1686
+ currentHiddenBuffer,
1687
+ numTokens,
1688
+ getLogitsWeights(this.#state),
1689
+ getLogitsConfig(this.#state),
1690
+ this.#state.debugFlags,
1691
+ this.#state.operatorDiagnostics
1692
+ );
1693
+ if (!logitsResult?.logitsBuffer) {
1694
+ throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
1695
+ }
1696
+ logitsBuffer = logitsResult.logitsBuffer;
1697
+ if (currentHiddenBuffer) {
1698
+ releaseBuffer(currentHiddenBuffer);
1699
+ currentHiddenBuffer = null;
1700
+ }
1701
+ if (logitsResult.logitsDtype !== 'f32') {
1702
+ throw new Error(
1703
+ `[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
1704
+ );
1705
+ }
1706
+ if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
1707
+ throw new Error(
1708
+ `[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
1709
+ `got ${logitsResult.vocabSize}.`
1710
+ );
1711
+ }
1712
+
1713
+ statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
1714
+ canvasLength: canvasIds.length,
1715
+ vocabSize: logitsResult.vocabSize,
1716
+ temperature,
1717
+ padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
1718
+ logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
1719
+ });
1720
+ const [argmaxData, entropyData] = await Promise.all([
1721
+ readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
1722
+ readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
1723
+ ]);
1724
+ const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
1725
+ const entropies = new Float32Array(entropyData);
1726
+ const logitsState = {
1727
+ logitsBuffer,
1728
+ logitsDtype: logitsResult.logitsDtype,
1729
+ vocabSize: logitsResult.vocabSize,
1730
+ canvasLength: canvasIds.length,
1731
+ temperature,
1732
+ releaseOnUse: true,
1733
+ };
1734
+ const selfConditioningState =
1735
+ await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
1736
+ logitsState,
1737
+ canvasIds.length,
1738
+ this.#state.modelConfig.hiddenSize,
1739
+ logitsResult.vocabSize
1740
+ );
1741
+ if (selfConditioningState) {
1742
+ releaseBuffer(logitsBuffer);
1743
+ logitsBuffer = null;
1744
+ } else {
1745
+ const returnedLogitsBuffer = logitsBuffer;
1746
+ logitsBuffer = null;
1747
+ logitsState.release = () => {
1748
+ releaseBuffer(returnedLogitsBuffer);
1749
+ };
1750
+ }
1751
+ return {
1752
+ argmaxCanvas,
1753
+ entropies,
1754
+ selfConditioningLogits: selfConditioningState ?? logitsState,
1755
+ };
1756
+ } finally {
1757
+ this.#state.currentSeqLen = seqLenBefore;
1758
+ if (statsBuffers?.argmaxBuffer) {
1759
+ releaseBuffer(statsBuffers.argmaxBuffer);
1760
+ }
1761
+ if (statsBuffers?.entropyBuffer) {
1762
+ releaseBuffer(statsBuffers.entropyBuffer);
1763
+ }
1764
+ if (logitsBuffer) {
1765
+ releaseBuffer(logitsBuffer);
1766
+ }
1767
+ if (currentHiddenBuffer) {
1768
+ releaseBuffer(currentHiddenBuffer);
1769
+ }
1770
+ if (selfConditioned?.buffer) {
1771
+ releaseBuffer(selfConditioned.buffer);
1772
+ }
1773
+ }
1774
+ }
1280
1775
 
1281
1776
  async *generate(prompt, options = {}) {
1282
1777
  yield* this._generateTokensInternal(prompt, options, 'text');
@@ -1297,10 +1792,21 @@ export class PipelineGenerator {
1297
1792
  this.#state.stats.gpuTimePrefillMs = undefined;
1298
1793
  this.#state.stats.gpuTimeDecodeMs = undefined;
1299
1794
  this.#state.stats.decodeRecordMs = 0;
1795
+ this.#state.stats.decodeRecordOps = 0;
1796
+ this.#state.stats.decodeRecordPasses = 0;
1797
+ this.#state.stats.decodeRecordOpLabels = {};
1300
1798
  this.#state.stats.decodeSubmitWaitMs = 0;
1301
1799
  this.#state.stats.decodeReadbackWaitMs = 0;
1800
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
1801
+ this.#state.stats.decodeReadbackCleanupMs = 0;
1802
+ this.#state.stats.decodeReadbackCopyMs = 0;
1803
+ this.#state.stats.prefillRecordMs = 0;
1804
+ this.#state.stats.prefillSubmitWaitMs = 0;
1302
1805
  this.#state.stats.singleTokenSubmitWaitMs = 0;
1303
1806
  this.#state.stats.singleTokenReadbackWaitMs = 0;
1807
+ this.#state.stats.singleTokenReadbackMapWaitMs = 0;
1808
+ this.#state.stats.singleTokenReadbackCleanupMs = 0;
1809
+ this.#state.stats.singleTokenReadbackCopyMs = 0;
1304
1810
  this.#state.stats.singleTokenOrchestrationMs = 0;
1305
1811
  this.#state.stats.pleHotVocabularyHits = 0;
1306
1812
  this.#state.stats.pleHotVocabularyMisses = 0;
@@ -1310,18 +1816,23 @@ export class PipelineGenerator {
1310
1816
  this.#state.stats.plePreparedTokenCacheBytes = 0;
1311
1817
  this.#state.stats.decodeMode = null;
1312
1818
  this.#state.stats.batchGuardReason = null;
1819
+ this.#state.stats.stopReason = null;
1820
+ this.#state.stats.stopTokenId = null;
1313
1821
  this.#state.stats.ttftMs = 0;
1314
1822
  const startTime = performance.now();
1315
1823
  const opts = resolveGenerateOptions(this.#state, options);
1824
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
1316
1825
  // Validate and normalize sampling parameters through single source of truth
1317
1826
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
1318
1827
  opts.temperature = samplingConfig.temperature;
1319
1828
  opts.topP = samplingConfig.topP;
1320
1829
  opts.topK = samplingConfig.topK;
1321
1830
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
1831
+ opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
1322
1832
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
1323
1833
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
1324
- if (diagnosticsEnabled) {
1834
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
1835
+ if (diagnosticsEnabled || tsirFixtureCfg) {
1325
1836
  const captureConfig = {
1326
1837
  ...createDefaultCaptureConfig(),
1327
1838
  enabled: true,
@@ -1330,13 +1841,20 @@ export class PipelineGenerator {
1330
1841
  };
1331
1842
  validateCaptureConfig(captureConfig);
1332
1843
  this.#state.operatorDiagnostics = {
1333
- enabled: true,
1844
+ enabled: diagnosticsEnabled === true,
1334
1845
  captureConfig,
1335
- emitter: new OperatorEventEmitter({
1846
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
1336
1847
  modelHash: this.#state.manifest?.modelId ?? null,
1337
1848
  runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
1338
1849
  executionPlanHash: opts.executionPlan?.id ?? null,
1339
- }),
1850
+ }) : null,
1851
+ tsirFixture: tsirFixtureCfg ? {
1852
+ dir: tsirFixtureCfg.dir,
1853
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
1854
+ ? tsirFixtureCfg.layerFilter
1855
+ : null,
1856
+ records: [],
1857
+ } : null,
1340
1858
  };
1341
1859
  }
1342
1860
 
@@ -1388,6 +1906,7 @@ export class PipelineGenerator {
1388
1906
  tokenIds.push(tokenId);
1389
1907
  }
1390
1908
  } else {
1909
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1391
1910
  this.#state.stats.decodeTimeMs = 0;
1392
1911
  this.#state.stats.tokensGenerated = 1;
1393
1912
  this.#state.stats.decodeTokens = 1;
@@ -1676,8 +2195,14 @@ export class PipelineGenerator {
1676
2195
  this.#state.stats.prefillProfileSteps = [];
1677
2196
  this.#state.decodeRing?.reset();
1678
2197
  this.#state.stats.decodeRecordMs = 0;
2198
+ this.#state.stats.decodeRecordOps = 0;
2199
+ this.#state.stats.decodeRecordPasses = 0;
2200
+ this.#state.stats.decodeRecordOpLabels = {};
1679
2201
  this.#state.stats.decodeSubmitWaitMs = 0;
1680
2202
  this.#state.stats.decodeReadbackWaitMs = 0;
2203
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
2204
+ this.#state.stats.decodeReadbackCleanupMs = 0;
2205
+ this.#state.stats.decodeReadbackCopyMs = 0;
1681
2206
  this.#state.stats.ttftMs = 0;
1682
2207
  const startTime = performance.now();
1683
2208
 
@@ -1704,6 +2229,7 @@ export class PipelineGenerator {
1704
2229
  topK: opts.topK,
1705
2230
  padTokenId,
1706
2231
  seed: opts.seed,
2232
+ suppressTokenIds: opts.suppressTokenIds,
1707
2233
  });
1708
2234
 
1709
2235
  const decodeRuntime = {
@@ -1726,6 +2252,7 @@ export class PipelineGenerator {
1726
2252
  if (options.onToken) options.onToken(firstToken, firstText);
1727
2253
 
1728
2254
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
2255
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1729
2256
  this.#state.stats.decodeTimeMs = 0;
1730
2257
  this.#state.stats.tokensGenerated = 1;
1731
2258
  this.#state.stats.decodeTokens = 1;
@@ -1785,6 +2312,7 @@ export class PipelineGenerator {
1785
2312
  const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
1786
2313
  const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
1787
2314
  const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
2315
+ const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
1788
2316
  const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
1789
2317
  config: this.#state.modelConfig,
1790
2318
  weights: this.#state.weights,
@@ -1792,6 +2320,7 @@ export class PipelineGenerator {
1792
2320
  const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
1793
2321
  config: this.#state.modelConfig,
1794
2322
  weights: this.#state.weights,
2323
+ perLayerInputsSession: resolvedPerLayerInputsSession,
1795
2324
  });
1796
2325
  const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
1797
2326
  const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
@@ -1807,7 +2336,7 @@ export class PipelineGenerator {
1807
2336
  batchSize: executionPlan.batchSize,
1808
2337
  useGPU: this.#state.useGPU,
1809
2338
  gpuSamplingAvailable,
1810
- disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
2339
+ disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
1811
2340
  disableCommandBatching: executionPlan.disableCommandBatching,
1812
2341
  isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
1813
2342
  finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
@@ -1822,6 +2351,7 @@ export class PipelineGenerator {
1822
2351
  else if (!this.#state.useGPU) reason = 'no_gpu';
1823
2352
  else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
1824
2353
  else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
2354
+ else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
1825
2355
  else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
1826
2356
  else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
1827
2357
  else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
@@ -1852,7 +2382,10 @@ export class PipelineGenerator {
1852
2382
  }
1853
2383
 
1854
2384
  while (tokensGenerated < opts.maxTokens) {
1855
- if (options.signal?.aborted) break;
2385
+ if (options.signal?.aborted) {
2386
+ this._recordStopReason('aborted');
2387
+ break;
2388
+ }
1856
2389
  if (this._hasFinitenessFallbackWindow() && useBatchPath) {
1857
2390
  useBatchPath = false;
1858
2391
  }
@@ -1864,21 +2397,58 @@ export class PipelineGenerator {
1864
2397
  hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
1865
2398
  hasGpuSplitPerLayerInputs,
1866
2399
  hasLinearAttentionLayers: hasLinearLayers,
2400
+ modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
2401
+ activationDtype: executionPlan.activationDtype,
2402
+ currentSeqLen: this.#state.currentSeqLen,
2403
+ maxDecodeTokens: opts.maxTokens,
2404
+ numLayers: this.#state.modelConfig.numLayers,
2405
+ hiddenSize: this.#state.modelConfig.hiddenSize,
1867
2406
  });
1868
2407
  const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
1869
2408
  const boundedBatchTokens = maxBatchDecodeTokens == null
1870
2409
  ? requestedBatchTokens
1871
2410
  : Math.min(requestedBatchTokens, maxBatchDecodeTokens);
1872
2411
  const thisBatchSize = Math.min(boundedBatchTokens, remaining);
2412
+ this.#state.batchingStats.requestedBatchTokens = Math.max(
2413
+ this.#state.batchingStats.requestedBatchTokens ?? 0,
2414
+ requestedBatchTokens
2415
+ );
2416
+ this.#state.batchingStats.effectiveBatchTokens = Math.max(
2417
+ this.#state.batchingStats.effectiveBatchTokens ?? 0,
2418
+ thisBatchSize
2419
+ );
2420
+ if (maxBatchDecodeTokens != null) {
2421
+ this.#state.batchingStats.maxBatchTokenCap = Math.max(
2422
+ this.#state.batchingStats.maxBatchTokenCap ?? 0,
2423
+ maxBatchDecodeTokens
2424
+ );
2425
+ }
2426
+ if (thisBatchSize < requestedBatchTokens) {
2427
+ this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
2428
+ }
1873
2429
  const lastToken = generatedIds[generatedIds.length - 1];
2430
+ const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
2431
+ ? {
2432
+ ...executionPlan,
2433
+ batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
2434
+ readbackInterval: readbackInterval == null
2435
+ ? null
2436
+ : Math.min(intervalBatches, thisBatchSize),
2437
+ }
2438
+ : executionPlan;
2439
+ const batchOpts = boundedExecutionPlan === executionPlan
2440
+ ? opts
2441
+ : { ...opts, executionPlan: boundedExecutionPlan };
1874
2442
 
1875
2443
  try {
1876
- const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, opts);
2444
+ const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
1877
2445
  let batchTokens = [];
1878
2446
  let hitStop = false;
2447
+ let stopTokenId = null;
1879
2448
  for (const tokenId of batchResult.tokens) {
1880
2449
  if (isStopToken(tokenId, stopTokenIds, eosToken)) {
1881
2450
  hitStop = true;
2451
+ stopTokenId = tokenId;
1882
2452
  break;
1883
2453
  }
1884
2454
  generatedIds.push(tokenId);
@@ -1901,9 +2471,15 @@ export class PipelineGenerator {
1901
2471
  if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
1902
2472
  if (opts.stopSequences.length > 0) {
1903
2473
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1904
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
2474
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2475
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
2476
+ break;
2477
+ }
2478
+ }
2479
+ if (hitStop) {
2480
+ this._recordStopReason('stop-token', stopTokenId);
2481
+ break;
1905
2482
  }
1906
- if (hitStop) break;
1907
2483
  if (shouldDisableBatchDecodeAfterShortBatch({
1908
2484
  hitStop,
1909
2485
  actualCount: batchResult.actualCount,
@@ -1941,7 +2517,10 @@ export class PipelineGenerator {
1941
2517
  if (options.onToken) options.onToken(nextToken, tokenText);
1942
2518
  }
1943
2519
  this._consumeFinitenessFallbackToken(opts);
1944
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
2520
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
2521
+ this._recordStopReason('stop-token', nextToken);
2522
+ break;
2523
+ }
1945
2524
  }
1946
2525
  } else if (opts.speculation?.mode === 'self') {
1947
2526
  // Self-speculation: decode one base token plus a configurable burst of
@@ -1979,11 +2558,17 @@ export class PipelineGenerator {
1979
2558
  }
1980
2559
  this._consumeFinitenessFallbackToken(opts);
1981
2560
 
1982
- if (isStopToken(baseToken, stopTokenIds, eosToken)) break;
2561
+ if (isStopToken(baseToken, stopTokenIds, eosToken)) {
2562
+ this._recordStopReason('stop-token', baseToken);
2563
+ break;
2564
+ }
1983
2565
  if (tokensGenerated >= opts.maxTokens) break;
1984
2566
  if (opts.stopSequences.length > 0) {
1985
2567
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1986
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
2568
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2569
+ this._recordStopReason('stop-sequence', baseToken);
2570
+ break;
2571
+ }
1987
2572
  }
1988
2573
 
1989
2574
  for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
@@ -2022,19 +2607,27 @@ export class PipelineGenerator {
2022
2607
  }
2023
2608
 
2024
2609
  if (isStopToken(specToken, stopTokenIds, eosToken)) {
2610
+ this._recordStopReason('stop-token', specToken);
2025
2611
  break;
2026
2612
  }
2027
2613
  if (opts.stopSequences.length > 0) {
2028
2614
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2029
2615
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2616
+ this._recordStopReason('stop-sequence', specToken);
2030
2617
  break;
2031
2618
  }
2032
2619
  }
2033
2620
  }
2034
- if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) break;
2621
+ if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
2622
+ this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
2623
+ break;
2624
+ }
2035
2625
  if (opts.stopSequences.length > 0) {
2036
2626
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2037
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
2627
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2628
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
2629
+ break;
2630
+ }
2038
2631
  }
2039
2632
  } else {
2040
2633
  const tokenStart = performance.now();
@@ -2088,17 +2681,27 @@ export class PipelineGenerator {
2088
2681
  log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
2089
2682
  }
2090
2683
 
2091
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
2684
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
2685
+ this._recordStopReason('stop-token', nextToken);
2686
+ break;
2687
+ }
2092
2688
  if (opts.stopSequences.length > 0) {
2093
2689
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2094
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
2690
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
2691
+ this._recordStopReason('stop-sequence', nextToken);
2692
+ break;
2693
+ }
2095
2694
  }
2096
2695
  }
2097
2696
  }
2098
2697
 
2698
+ if (!this.#state.stats.stopReason) {
2699
+ this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
2700
+ }
2099
2701
  this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
2100
2702
  this.#state.stats.tokensGenerated = tokensGenerated;
2101
2703
  this.#state.stats.decodeTokens = tokensGenerated;
2704
+ this.#state.stats.batching = { ...this.#state.batchingStats };
2102
2705
  }
2103
2706
 
2104
2707
  async _prefillToHidden(inputIds, opts) {
@@ -2111,6 +2714,8 @@ export class PipelineGenerator {
2111
2714
  const numTokens = inputIds.length;
2112
2715
  const config = this.#state.modelConfig;
2113
2716
  const startPos = this.#state.currentSeqLen;
2717
+ const tracePrefillEnabled = isTraceEnabled('perf');
2718
+ const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
2114
2719
  const returnHidden = opts?._returnHidden === true;
2115
2720
  const embeddingInputIds = resolvePrefillEmbeddingInputIds(
2116
2721
  inputIds,
@@ -2146,13 +2751,14 @@ export class PipelineGenerator {
2146
2751
  }
2147
2752
 
2148
2753
  const embedBufferRaw = this.#state.weights.get('embed');
2149
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
2754
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
2150
2755
  throw new Error('Embed buffer not found or not a supported buffer type');
2151
2756
  }
2152
2757
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
2153
2758
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
2154
2759
  ? embedBufferRaw.dtype
2155
2760
  : getWeightDtype(embedBufferRaw);
2761
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
2156
2762
  if (opts.debug) {
2157
2763
  const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
2158
2764
  log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
@@ -2167,7 +2773,9 @@ export class PipelineGenerator {
2167
2773
  );
2168
2774
  const createRecorder = (label) => {
2169
2775
  if (!device || disableCommandBatching) return undefined;
2170
- return opts.profile ? createProfilingRecorder(label) : createCommandRecorder(label);
2776
+ return opts.profile
2777
+ ? createProfilingRecorder(label, device)
2778
+ : createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
2171
2779
  };
2172
2780
  const recorder = createRecorder('prefill');
2173
2781
  const debugCheckBuffer = this.#state.debug
@@ -2183,6 +2791,7 @@ export class PipelineGenerator {
2183
2791
  opts.executionPlan
2184
2792
  );
2185
2793
  context.currentTokenIds = inputIds;
2794
+ context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
2186
2795
  context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
2187
2796
  ? null
2188
2797
  : {
@@ -2232,6 +2841,16 @@ export class PipelineGenerator {
2232
2841
 
2233
2842
  const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
2234
2843
  const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
2844
+ if (tracePrefillEnabled) {
2845
+ trace.perf('Prefill phase start', {
2846
+ numTokens,
2847
+ startPos,
2848
+ numLayers: config.numLayers,
2849
+ activationDtype,
2850
+ returnHidden,
2851
+ });
2852
+ }
2853
+ const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
2235
2854
  let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
2236
2855
  hiddenSize: config.hiddenSize,
2237
2856
  vocabSize: config.vocabSize,
@@ -2242,18 +2861,29 @@ export class PipelineGenerator {
2242
2861
  debugProbes: this.#state.runtimeConfig.shared.debug.probes,
2243
2862
  operatorDiagnostics: this.#state.operatorDiagnostics,
2244
2863
  activationDtype,
2245
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
2864
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
2865
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
2246
2866
  executionPolicies: this.#state.executionV1State?.policies ?? null,
2247
2867
  });
2868
+ if (tracePrefillEnabled) {
2869
+ trace.perf('Prefill embed complete', {
2870
+ numTokens,
2871
+ elapsedMs: performance.now() - embedTraceStart,
2872
+ });
2873
+ }
2248
2874
  let hiddenStates = baseEmbeddings;
2249
2875
  let perLayerInputs = null;
2876
+ const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
2250
2877
  try {
2251
2878
  hiddenStates = await applyPrefixEmbeddingOverride(
2252
2879
  baseEmbeddings,
2253
2880
  embeddingOverride,
2254
2881
  config.hiddenSize,
2255
2882
  '_prefill',
2256
- this.#state.executionV1State?.policies ?? null
2883
+ {
2884
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
2885
+ transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
2886
+ }
2257
2887
  );
2258
2888
  perLayerInputs = await preparePerLayerInputs(
2259
2889
  embeddingInputIds,
@@ -2264,6 +2894,13 @@ export class PipelineGenerator {
2264
2894
  pleCache: this.#state.pleCache ?? null,
2265
2895
  }
2266
2896
  );
2897
+ if (tracePrefillEnabled) {
2898
+ trace.perf('Prefill per-layer inputs complete', {
2899
+ numTokens,
2900
+ elapsedMs: performance.now() - perLayerInputsTraceStart,
2901
+ materialized: Array.isArray(perLayerInputs),
2902
+ });
2903
+ }
2267
2904
  } catch (error) {
2268
2905
  if (isGpuBufferInstance(hiddenStates?.buffer)) {
2269
2906
  releaseBuffer(hiddenStates.buffer);
@@ -2312,7 +2949,7 @@ export class PipelineGenerator {
2312
2949
  if (this.#state.finitenessBuffer) {
2313
2950
  const device = getDevice();
2314
2951
  if (device) {
2315
- device.queue.writeBuffer(this.#state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
2952
+ device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
2316
2953
  }
2317
2954
  }
2318
2955
 
@@ -2321,16 +2958,37 @@ export class PipelineGenerator {
2321
2958
  // Chunked recorder submission: submit every N layers to release tracked intermediate
2322
2959
  // buffers, preventing unbounded memory growth during large prefills. Critical for
2323
2960
  // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2324
- const PREFILL_RECORDER_CHUNK_LAYERS = 4;
2961
+ const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
2962
+ hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
2963
+ numTokens,
2964
+ });
2325
2965
 
2326
2966
  let currentHiddenBuffer = hiddenStates.buffer;
2967
+ let prefillRecordMs = 0;
2968
+ let prefillSubmitWaitMs = 0;
2969
+ const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
2970
+ const traceLayerHealthEnabled = isTraceEnabled('logits');
2327
2971
  try {
2328
2972
  for (let l = 0; l < config.numLayers; l++) {
2973
+ // Per-layer hard cancellation: when the caller's AbortSignal aborts,
2974
+ // exit the prefill loop between layer dispatches rather than
2975
+ // continuing to burn GPU on superseded work. Granularity is one
2976
+ // layer (~5-50ms), which is the fastest cancel granularity WebGPU
2977
+ // exposes today.
2978
+ if (opts?.signal?.aborted) {
2979
+ const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
2980
+ const err = new Error(reason);
2981
+ err.name = "AbortError";
2982
+ err.code = "ABORT_ERR";
2983
+ throw err;
2984
+ }
2329
2985
  context.recorder = currentRecorder;
2330
2986
  context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
2331
2987
 
2332
2988
  const prevBuffer = currentHiddenBuffer;
2989
+ const layerRecordStart = performance.now();
2333
2990
  const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
2991
+ prefillRecordMs += performance.now() - layerRecordStart;
2334
2992
  if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
2335
2993
  currentHiddenBuffer = layerOutput;
2336
2994
  releasePerLayerInputBuffer(
@@ -2346,9 +3004,10 @@ export class PipelineGenerator {
2346
3004
 
2347
3005
  const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
2348
3006
  const isChunkBoundary = !isCheckpoint
3007
+ && !traceLayerHealthEnabled
2349
3008
  && currentRecorder
2350
3009
  && l < config.numLayers - 1
2351
- && (l + 1) % PREFILL_RECORDER_CHUNK_LAYERS === 0;
3010
+ && (l + 1) % prefillRecorderChunkLayers === 0;
2352
3011
 
2353
3012
  if (isCheckpoint && currentRecorder) {
2354
3013
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
@@ -2361,6 +3020,27 @@ export class PipelineGenerator {
2361
3020
  currentRecorder = undefined;
2362
3021
  }
2363
3022
 
3023
+ if (traceLayerHealthEnabled && currentRecorder) {
3024
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3025
+ currentHiddenBuffer,
3026
+ currentRecorder,
3027
+ 'prefill_trace_layer_health_carry'
3028
+ );
3029
+ const traceSubmitStart = performance.now();
3030
+ await currentRecorder.submitAndWait();
3031
+ await recordProfile(currentRecorder);
3032
+ prefillSubmitWaitMs += performance.now() - traceSubmitStart;
3033
+ await traceActivationHealth(
3034
+ `PREFILL_LAYER_${l}_HEALTH`,
3035
+ currentHiddenBuffer,
3036
+ activationDtype,
3037
+ numTokens * config.hiddenSize
3038
+ );
3039
+ currentRecorder = l < config.numLayers - 1
3040
+ ? createRecorder('prefill-trace')
3041
+ : undefined;
3042
+ }
3043
+
2364
3044
  const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
2365
3045
  if (shouldDebug && !currentRecorder) {
2366
3046
  const device = getDevice();
@@ -2408,11 +3088,50 @@ export class PipelineGenerator {
2408
3088
  currentRecorder,
2409
3089
  'prefill_chunk_carry'
2410
3090
  );
2411
- await currentRecorder.submitAndWait();
2412
- await recordProfile(currentRecorder);
3091
+ // Chunk boundary exists only to bound intermediate buffer lifetime.
3092
+ // When the runtime opts into async chunk submit AND profile timings
3093
+ // are not being collected, skip the CPU-GPU wait: queue order is
3094
+ // preserved across submits and deferred cleanup still releases
3095
+ // tracked buffers when GPU work completes. Profile runs keep the
3096
+ // sync path because resolveProfileTimings requires the submitted
3097
+ // work to be done.
3098
+ const chunkSubmitMode = resolvePrefillChunkSubmitMode(
3099
+ this.#state.runtimeConfig,
3100
+ this.#state.modelConfig
3101
+ );
3102
+ const chunkSubmitStart = performance.now();
3103
+ if (chunkSubmitMode === 'async' && !opts.profile) {
3104
+ currentRecorder.submit({ cleanup: 'deferred' });
3105
+ } else {
3106
+ await currentRecorder.submitAndWait();
3107
+ await recordProfile(currentRecorder);
3108
+ }
3109
+ prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
3110
+ await traceActivationHealth(
3111
+ `PREFILL_LAYER_${l}_HEALTH`,
3112
+ currentHiddenBuffer,
3113
+ activationDtype,
3114
+ numTokens * config.hiddenSize
3115
+ );
3116
+ if (tracePrefillEnabled) {
3117
+ trace.perf('Prefill chunk submitted', {
3118
+ layer: l,
3119
+ elapsedMs: performance.now() - layerLoopTraceStart,
3120
+ prefillRecordMs,
3121
+ prefillSubmitWaitMs,
3122
+ });
3123
+ }
2413
3124
  currentRecorder = createRecorder('prefill-chunk');
2414
3125
  }
2415
3126
  }
3127
+ if (tracePrefillEnabled) {
3128
+ trace.perf('Prefill layer loop recorded', {
3129
+ numTokens,
3130
+ elapsedMs: performance.now() - layerLoopTraceStart,
3131
+ prefillRecordMs,
3132
+ prefillSubmitWaitMs,
3133
+ });
3134
+ }
2416
3135
  } finally {
2417
3136
  context.perLayerInputBuffer = null;
2418
3137
  if (perLayerInputs) {
@@ -2426,8 +3145,26 @@ export class PipelineGenerator {
2426
3145
  }
2427
3146
  }
2428
3147
  releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
3148
+ this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
3149
+ this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
2429
3150
  }
2430
3151
 
3152
+ {
3153
+ const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
3154
+ if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
3155
+ if (currentRecorder) {
3156
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3157
+ currentHiddenBuffer,
3158
+ currentRecorder,
3159
+ 'prefill_tsir_drain_carry'
3160
+ );
3161
+ await currentRecorder.submitAndWait();
3162
+ await recordProfile(currentRecorder);
3163
+ currentRecorder = undefined;
3164
+ }
3165
+ await drainPendingTsirReads(tsirFixtureCfg);
3166
+ }
3167
+ }
2431
3168
  if (this.#state.finitenessBuffer) {
2432
3169
  if (currentRecorder) {
2433
3170
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
@@ -2506,6 +3243,7 @@ export class PipelineGenerator {
2506
3243
  && !this.#state.disableRecordedLogits
2507
3244
  && numTokens === 1;
2508
3245
  if (currentRecorder && canRecordLogits) {
3246
+ const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
2509
3247
  const recorded = await recordLogitsGPU(
2510
3248
  currentRecorder,
2511
3249
  currentHiddenBuffer,
@@ -2526,6 +3264,13 @@ export class PipelineGenerator {
2526
3264
  const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
2527
3265
  releaseBuffer(recorded.logitsBuffer);
2528
3266
  lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
3267
+ if (tracePrefillEnabled) {
3268
+ trace.perf('Prefill recorded logits complete', {
3269
+ numTokens,
3270
+ vocabSize: logitsVocabSize,
3271
+ elapsedMs: performance.now() - logitsTraceStart,
3272
+ });
3273
+ }
2529
3274
 
2530
3275
  const health = getLogitsHealth(lastLogits);
2531
3276
  if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
@@ -2564,6 +3309,7 @@ export class PipelineGenerator {
2564
3309
 
2565
3310
  releaseBuffer(currentHiddenBuffer);
2566
3311
  } else {
3312
+ const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
2567
3313
  if (currentRecorder) {
2568
3314
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2569
3315
  currentHiddenBuffer,
@@ -2591,6 +3337,13 @@ export class PipelineGenerator {
2591
3337
  ? logits
2592
3338
  : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
2593
3339
  releaseBuffer(currentHiddenBuffer);
3340
+ if (tracePrefillEnabled) {
3341
+ trace.perf('Prefill logits complete', {
3342
+ numTokens,
3343
+ vocabSize: logitsVocabSize,
3344
+ elapsedMs: performance.now() - logitsTraceStart,
3345
+ });
3346
+ }
2594
3347
  }
2595
3348
 
2596
3349
  this.#state.currentSeqLen = startPos + numTokens;
@@ -2610,6 +3363,9 @@ export class PipelineGenerator {
2610
3363
  if (opts.debug) {
2611
3364
  logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
2612
3365
  }
3366
+ if (isTraceEnabled('logits')) {
3367
+ trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3368
+ }
2613
3369
 
2614
3370
  if (opts.debug) {
2615
3371
  if (this.#state.kvCache?.hasGPUCache?.()) {
@@ -2619,6 +3375,13 @@ export class PipelineGenerator {
2619
3375
  }
2620
3376
  }
2621
3377
 
3378
+ if (tracePrefillEnabled) {
3379
+ trace.perf('Prefill phase complete', {
3380
+ numTokens,
3381
+ totalMs: performance.now() - prefillTraceStart,
3382
+ });
3383
+ }
3384
+
2622
3385
  return lastLogits;
2623
3386
  }
2624
3387