doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -7,7 +7,18 @@ import { markWarmed as markKernelCacheWarmed } from '../../../gpu/kernel-selecti
7
7
  import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
8
8
  import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
9
9
  import { allowReadback } from '../../../gpu/perf-guards.js';
10
- import { log, trace } from '../../../debug/index.js';
10
+ import { log, trace, isTraceEnabled } from '../../../debug/index.js';
11
+ import {
12
+ runMatmul,
13
+ runRMSNorm,
14
+ runGeLU,
15
+ runResidualAdd,
16
+ runScale,
17
+ runSoftmax,
18
+ runSoftEmbeddingSplitF16,
19
+ runSoftEmbeddingLogitsF16,
20
+ } from '../../../gpu/kernel-selector.js';
21
+ import { runDiffusionGemmaCanvasStats } from '../../../gpu/kernels/diffusion-gemma-sampling.js';
11
22
  import {
12
23
  CAPTURE_LEVELS,
13
24
  createDefaultCaptureConfig,
@@ -21,9 +32,9 @@ import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './samplin
21
32
  import { createKVCache, isStopToken } from './init.js';
22
33
  import { embed } from './embed.js';
23
34
  import { processLayer } from './layer.js';
24
- import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
35
+ import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
25
36
  import { OperatorEventEmitter } from './operator-events.js';
26
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
37
+ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype, getWeightMetadata, getLayout } from '../../../gpu/weight-buffer.js';
27
38
  import {
28
39
  decodeStep,
29
40
  decodeStepLogits,
@@ -84,6 +95,11 @@ import {
84
95
  hasRangeBackedPerLayerInputEmbeddings,
85
96
  } from './per-layer-inputs.js';
86
97
  import { createTensor } from '../../../gpu/tensor.js';
98
+ import { getQKNormOnesBuffer } from './attention/types.js';
99
+ import {
100
+ getWeightBuffer as getPipelineWeightBuffer,
101
+ getNormWeightBuffer as getPipelineNormWeightBuffer,
102
+ } from './weights.js';
87
103
 
88
104
  // Extracted standalone helpers
89
105
  import {
@@ -93,7 +109,9 @@ import {
93
109
  resolvePrefillEmbeddingInputIds,
94
110
  resolvePrefillMultimodalBidirectionalSpan,
95
111
  applyPrefixEmbeddingOverride,
112
+ resolvePrefixEmbeddingOverrideTransitionDeclaredBy,
96
113
  shouldDisablePrefillCommandBatching,
114
+ resolveEffectivePrefillTokenChunkSize,
97
115
  } from './generator-prefill-helpers.js';
98
116
  import {
99
117
  shouldDisableBatchDecodeAfterShortBatch,
@@ -111,6 +129,260 @@ import {
111
129
  cloneRuntimeInferenceWithKVDtype,
112
130
  } from './generator-decode-policy.js';
113
131
 
132
+ const SPECIAL_LIKE_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>\n]{1,32}>)$/i;
133
+ const FINITENESS_RESET_WORDS = new Uint32Array(4);
134
+ const tokenizerSuppressionCache = new WeakMap();
135
+ const PREFILL_CHUNK_SUBMIT_MODES = new Set(['sync', 'async']);
136
+
137
+ export function resolvePrefillChunkSubmitMode(runtimeConfig, modelConfig) {
138
+ const runtimeSubmit = runtimeConfig?.inference?.session?.prefillChunkSubmitMode;
139
+ const manifestSubmit = modelConfig?.sessionSettings?.prefillChunkSubmitMode;
140
+ const submit = (runtimeSubmit !== undefined && runtimeSubmit !== null)
141
+ ? runtimeSubmit
142
+ : manifestSubmit;
143
+ if (submit === undefined || submit === null) {
144
+ throw new Error('[Pipeline] runtime.inference.session.prefillChunkSubmitMode is required.');
145
+ }
146
+ if (!PREFILL_CHUNK_SUBMIT_MODES.has(submit)) {
147
+ throw new Error(
148
+ `[Pipeline] runtime.inference.session.prefillChunkSubmitMode must be "sync" or "async"; got "${String(submit)}".`
149
+ );
150
+ }
151
+ return submit;
152
+ }
153
+
154
+ function getTokenizerSuppressionCache(tokenizer) {
155
+ let cache = tokenizerSuppressionCache.get(tokenizer);
156
+ if (!cache) {
157
+ cache = new Map();
158
+ tokenizerSuppressionCache.set(tokenizer, cache);
159
+ }
160
+ return cache;
161
+ }
162
+
163
+ function decodeSingleTokenForSuppression(tokenizer, tokenId) {
164
+ if (!tokenizer || typeof tokenizer.decode !== 'function') {
165
+ return '';
166
+ }
167
+ try {
168
+ return tokenizer.decode([tokenId], false, false);
169
+ } catch {
170
+ return '';
171
+ }
172
+ }
173
+
174
+ function collectTokenizerSuppressedTokenIds(tokenizer, vocabSize, samplingConfig) {
175
+ if (!tokenizer || !Number.isInteger(vocabSize) || vocabSize < 1) {
176
+ return [];
177
+ }
178
+ if (samplingConfig.suppressSpecialTokens !== true && samplingConfig.suppressSpecialLikeTokens !== true) {
179
+ return [];
180
+ }
181
+ const cache = getTokenizerSuppressionCache(tokenizer);
182
+ const cacheKey = [
183
+ vocabSize,
184
+ samplingConfig.suppressSpecialTokens === true ? 'special' : 'plain',
185
+ samplingConfig.suppressSpecialLikeTokens === true ? 'specialLike' : 'literal',
186
+ ].join(':');
187
+ const cached = cache.get(cacheKey);
188
+ if (cached) {
189
+ return cached;
190
+ }
191
+ const tokenIds = [];
192
+ for (let tokenId = 0; tokenId < vocabSize; tokenId++) {
193
+ const tokenizerSpecial = samplingConfig.suppressSpecialTokens === true
194
+ && typeof tokenizer.isSpecialToken === 'function'
195
+ && tokenizer.isSpecialToken(tokenId);
196
+ const specialLike = samplingConfig.suppressSpecialLikeTokens === true
197
+ && SPECIAL_LIKE_TOKEN_RE.test(decodeSingleTokenForSuppression(tokenizer, tokenId).trim());
198
+ if (tokenizerSpecial || specialLike) {
199
+ tokenIds.push(tokenId);
200
+ }
201
+ }
202
+ cache.set(cacheKey, tokenIds);
203
+ return tokenIds;
204
+ }
205
+
206
+ function resolveSuppressedSamplingTokenIds(state, samplingConfig) {
207
+ const vocabSize = state.modelConfig?.vocabSize;
208
+ const suppressed = new Set(samplingConfig.suppressTokenIds);
209
+ const stopTokenIds = new Set(state.modelConfig?.stopTokenIds ?? []);
210
+ const eosToken = state.tokenizer?.getSpecialTokens?.()?.eos;
211
+ if (Number.isInteger(eosToken)) {
212
+ stopTokenIds.add(eosToken);
213
+ }
214
+ for (const tokenId of collectTokenizerSuppressedTokenIds(state.tokenizer, vocabSize, samplingConfig)) {
215
+ if (!stopTokenIds.has(tokenId)) {
216
+ suppressed.add(tokenId);
217
+ }
218
+ }
219
+ return [...suppressed];
220
+ }
221
+
222
+ async function traceActivationHealth(label, buffer, dtype, elementCount) {
223
+ if (!isTraceEnabled('logits') || !isGpuBufferInstance(buffer)) {
224
+ return;
225
+ }
226
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
227
+ const data = await readBuffer(buffer, elementCount * bytesPerElement);
228
+ trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
229
+ }
230
+
231
+ function ownsBorrowedWeightBuffer(weight) {
232
+ return !isGpuBufferInstance(weight) && !isWeightBuffer(weight);
233
+ }
234
+
235
+ function borrowLinearWeight(weight, label) {
236
+ if (!weight) {
237
+ throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
238
+ }
239
+ if (isSplitWeightBuffer(weight)) {
240
+ throw new Error(
241
+ `DiffusionGemma self-conditioning does not support split weight storage for ${label}.`
242
+ );
243
+ }
244
+ return {
245
+ value: getPipelineWeightBuffer(weight, label),
246
+ owned: ownsBorrowedWeightBuffer(weight),
247
+ };
248
+ }
249
+
250
+ function borrowNormWeight(weight, label) {
251
+ if (!weight) {
252
+ throw new Error(`DiffusionGemma self-conditioning missing ${label}.`);
253
+ }
254
+ if (isSplitWeightBuffer(weight)) {
255
+ throw new Error(
256
+ `DiffusionGemma self-conditioning does not support split norm storage for ${label}.`
257
+ );
258
+ }
259
+ return {
260
+ value: getPipelineNormWeightBuffer(weight, label),
261
+ owned: ownsBorrowedWeightBuffer(weight),
262
+ };
263
+ }
264
+
265
+ function releaseBorrowedWeight(borrowed) {
266
+ if (!borrowed?.owned) {
267
+ return;
268
+ }
269
+ const value = borrowed.value;
270
+ releaseBuffer(isWeightBuffer(value) ? value.buffer : value);
271
+ }
272
+
273
+ function canUseChunkedSoftEmbeddingLogits(logitsState, embeddingWeight, embeddingTranspose) {
274
+ return logitsState != null
275
+ && isWeightBuffer(embeddingWeight)
276
+ && getWeightDtype(embeddingWeight) === 'f16'
277
+ && getLayout(embeddingWeight) === 'row'
278
+ && embeddingTranspose !== true;
279
+ }
280
+
281
+ function resolveDiffusionGemmaSoftEmbeddingChunkRows(runtimeConfig) {
282
+ const value = runtimeConfig?.inference?.diffusionGemma?.softEmbeddingLogitsChunkRows;
283
+ if (value == null) {
284
+ return undefined;
285
+ }
286
+ if (!Number.isInteger(value) || value <= 0) {
287
+ throw new Error(
288
+ 'runtime.inference.diffusionGemma.softEmbeddingLogitsChunkRows must be a positive integer.'
289
+ );
290
+ }
291
+ return value;
292
+ }
293
+
294
+ function normalizeCanvasTokenIds(canvas, context) {
295
+ if (!Array.isArray(canvas) && !ArrayBuffer.isView(canvas)) {
296
+ throw new Error(`[DiffusionGemma] ${context}.canvas must be an array or typed array of token IDs.`);
297
+ }
298
+ return Array.from(canvas, (value, index) => {
299
+ if (!Number.isFinite(value) || Math.floor(value) !== value || value < 0) {
300
+ throw new Error(`[DiffusionGemma] ${context}.canvas[${index}] must be a non-negative integer token ID.`);
301
+ }
302
+ return value;
303
+ });
304
+ }
305
+
306
+ function normalizeSelfConditioningLogits(logits, canvasLength, vocabSize) {
307
+ if (logits == null) {
308
+ return null;
309
+ }
310
+ const expected = canvasLength * vocabSize;
311
+ const values = logits instanceof Float32Array
312
+ ? logits
313
+ : (Array.isArray(logits) ? Float32Array.from(logits) : null);
314
+ if (!values) {
315
+ throw new Error('[DiffusionGemma] selfConditioningLogits must be a Float32Array or number array.');
316
+ }
317
+ if (values.length !== expected) {
318
+ throw new Error(
319
+ `[DiffusionGemma] selfConditioningLogits length mismatch: expected ${expected}, got ${values.length}.`
320
+ );
321
+ }
322
+ return values;
323
+ }
324
+
325
+ function normalizeSelfConditioningLogitsState(logits, canvasLength, vocabSize) {
326
+ if (logits == null || !isGpuBufferInstance(logits.logitsBuffer)) {
327
+ return null;
328
+ }
329
+ const dtype = logits.logitsDtype;
330
+ if (dtype !== 'f32') {
331
+ throw new Error(`[DiffusionGemma] GPU selfConditioningLogits require f32 logits, got "${dtype}".`);
332
+ }
333
+ if (logits.vocabSize !== vocabSize) {
334
+ throw new Error(
335
+ `[DiffusionGemma] GPU selfConditioningLogits vocab mismatch: expected ${vocabSize}, got ${logits.vocabSize}.`
336
+ );
337
+ }
338
+ if (logits.canvasLength !== canvasLength) {
339
+ throw new Error(
340
+ `[DiffusionGemma] GPU selfConditioningLogits canvas mismatch: expected ${canvasLength}, got ${logits.canvasLength}.`
341
+ );
342
+ }
343
+ const temperature = logits.temperature ?? 1.0;
344
+ if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
345
+ throw new Error('[DiffusionGemma] GPU selfConditioningLogits temperature must be positive.');
346
+ }
347
+ return {
348
+ logitsBuffer: logits.logitsBuffer,
349
+ logitsDtype: dtype,
350
+ temperature,
351
+ releaseOnUse: logits.releaseOnUse !== false,
352
+ };
353
+ }
354
+
355
+ function normalizeSelfConditioningSoftEmbeddingState(state, canvasLength, hiddenSize) {
356
+ if (state?.kind !== 'soft_embedding') {
357
+ return null;
358
+ }
359
+ if (!isGpuBufferInstance(state.buffer)) {
360
+ throw new Error('[DiffusionGemma] GPU selfConditioning soft embedding requires a GPU buffer.');
361
+ }
362
+ const dtype = state.dtype ?? 'f32';
363
+ if (dtype !== 'f32') {
364
+ throw new Error(`[DiffusionGemma] GPU selfConditioning soft embedding requires f32 dtype, got "${dtype}".`);
365
+ }
366
+ if (state.canvasLength !== canvasLength) {
367
+ throw new Error(
368
+ `[DiffusionGemma] GPU selfConditioning soft embedding canvas mismatch: ` +
369
+ `expected ${canvasLength}, got ${state.canvasLength}.`
370
+ );
371
+ }
372
+ if (state.hiddenSize !== hiddenSize) {
373
+ throw new Error(
374
+ `[DiffusionGemma] GPU selfConditioning soft embedding hidden mismatch: ` +
375
+ `expected ${hiddenSize}, got ${state.hiddenSize}.`
376
+ );
377
+ }
378
+ return {
379
+ buffer: state.buffer,
380
+ dtype,
381
+ releaseOnUse: state.releaseOnUse !== false,
382
+ scaled: state.scaled === true,
383
+ };
384
+ }
385
+
114
386
  let intentBundleModulePromise = null;
115
387
 
116
388
  async function getExperimentalIntentBundleModule() {
@@ -256,8 +528,14 @@ export class PipelineGenerator {
256
528
  this.#state.stats.ttftMs = 0;
257
529
  this.#state.stats.decodeTimeMs = 0;
258
530
  this.#state.stats.decodeRecordMs = 0;
531
+ this.#state.stats.decodeRecordOps = 0;
532
+ this.#state.stats.decodeRecordPasses = 0;
533
+ this.#state.stats.decodeRecordOpLabels = {};
259
534
  this.#state.stats.decodeSubmitWaitMs = 0;
260
535
  this.#state.stats.decodeReadbackWaitMs = 0;
536
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
537
+ this.#state.stats.decodeReadbackCleanupMs = 0;
538
+ this.#state.stats.decodeReadbackCopyMs = 0;
261
539
  this.#state.decodeStepCount = 0;
262
540
  this.#state.disableRecordedLogits = false;
263
541
  this.#state.disableFusedDecode = false;
@@ -267,6 +545,12 @@ export class PipelineGenerator {
267
545
  totalBatchedTimeMs: 0,
268
546
  totalUnbatchedTimeMs: 0,
269
547
  gpuSubmissions: 0,
548
+ requestedBatchTokens: 0,
549
+ effectiveBatchTokens: 0,
550
+ executedBatchTokens: 0,
551
+ resolvedBatchTokens: 0,
552
+ maxBatchTokenCap: null,
553
+ batchClampCount: 0,
270
554
  };
271
555
  resetActiveExecutionPlan(this.#state);
272
556
  this.#state.decodeRing?.reset();
@@ -339,6 +623,7 @@ export class PipelineGenerator {
339
623
  topK: opts.topK,
340
624
  padTokenId,
341
625
  seed: opts.seed,
626
+ suppressTokenIds: opts.suppressTokenIds,
342
627
  });
343
628
  if (typeof opts.onLogits === 'function') {
344
629
  opts.onLogits(sampledLogits, {
@@ -349,6 +634,65 @@ export class PipelineGenerator {
349
634
  return tokenId;
350
635
  }
351
636
 
637
+ _resolvePrefillTokenChunkSize(inputIds) {
638
+ const chunkSize = resolveEffectivePrefillTokenChunkSize(this.#state);
639
+ if (chunkSize === undefined) {
640
+ throw new Error('inference.session.prefillTokenChunkSize is required; use null to disable token-chunked prefill.');
641
+ }
642
+ if (chunkSize === null) {
643
+ return null;
644
+ }
645
+ if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
646
+ throw new Error('inference.session.prefillTokenChunkSize must be null or a positive integer.');
647
+ }
648
+ return chunkSize < inputIds.length ? chunkSize : null;
649
+ }
650
+
651
+ async _commitPrefillHiddenChunk(prefillResult) {
652
+ const {
653
+ numTokens,
654
+ startPos,
655
+ currentRecorder,
656
+ recordProfile,
657
+ currentHiddenBuffer,
658
+ } = prefillResult;
659
+
660
+ try {
661
+ if (currentRecorder) {
662
+ await currentRecorder.submitAndWait();
663
+ await recordProfile(currentRecorder);
664
+ } else {
665
+ const device = getDevice();
666
+ if (device) {
667
+ await device.queue.onSubmittedWorkDone();
668
+ }
669
+ }
670
+ this.#state.currentSeqLen = startPos + numTokens;
671
+ } finally {
672
+ releaseBuffer(currentHiddenBuffer);
673
+ }
674
+ }
675
+
676
+ async _prefillInputIdsToLogits(inputIds, opts) {
677
+ const chunkSize = this._resolvePrefillTokenChunkSize(inputIds);
678
+ if (chunkSize === null) {
679
+ return this._prefill(inputIds, opts);
680
+ }
681
+
682
+ for (let offset = 0; offset < inputIds.length; offset += chunkSize) {
683
+ const end = Math.min(offset + chunkSize, inputIds.length);
684
+ const chunk = inputIds.slice(offset, end);
685
+ const isFinalChunk = end === inputIds.length;
686
+ if (isFinalChunk) {
687
+ return this._prefill(chunk, opts);
688
+ }
689
+ const prefillResult = await this._prefillToHidden(chunk, opts);
690
+ await this._commitPrefillHiddenChunk(prefillResult);
691
+ }
692
+
693
+ return this._prefill(inputIds, opts);
694
+ }
695
+
352
696
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
353
697
  const prefillStartSeqLen = this.#state.currentSeqLen;
354
698
  const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, contextLabel, opts.inputIds);
@@ -357,8 +701,9 @@ export class PipelineGenerator {
357
701
  }
358
702
 
359
703
  let logits;
704
+ const runPrefill = () => this._prefillInputIdsToLogits(inputIds, opts);
360
705
  try {
361
- logits = await this._prefill(inputIds, opts);
706
+ logits = await runPrefill();
362
707
  } catch (error) {
363
708
  if (!this._shouldUseFinitenessFallback(error, contextLabel)) {
364
709
  throw error;
@@ -368,7 +713,7 @@ export class PipelineGenerator {
368
713
  opts,
369
714
  contextLabel,
370
715
  opts.maxTokens ?? 1,
371
- () => this._prefill(inputIds, opts),
716
+ runPrefill,
372
717
  prefillStartSeqLen
373
718
  );
374
719
  }
@@ -423,12 +768,21 @@ export class PipelineGenerator {
423
768
  this.#state.stats.gpuTimePrefillMs = undefined;
424
769
  this.#state.stats.gpuTimeDecodeMs = undefined;
425
770
  this.#state.stats.decodeRecordMs = 0;
771
+ this.#state.stats.decodeRecordOps = 0;
772
+ this.#state.stats.decodeRecordPasses = 0;
773
+ this.#state.stats.decodeRecordOpLabels = {};
426
774
  this.#state.stats.decodeSubmitWaitMs = 0;
427
775
  this.#state.stats.decodeReadbackWaitMs = 0;
776
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
777
+ this.#state.stats.decodeReadbackCleanupMs = 0;
778
+ this.#state.stats.decodeReadbackCopyMs = 0;
428
779
  this.#state.stats.prefillRecordMs = 0;
429
780
  this.#state.stats.prefillSubmitWaitMs = 0;
430
781
  this.#state.stats.singleTokenSubmitWaitMs = 0;
431
782
  this.#state.stats.singleTokenReadbackWaitMs = 0;
783
+ this.#state.stats.singleTokenReadbackMapWaitMs = 0;
784
+ this.#state.stats.singleTokenReadbackCleanupMs = 0;
785
+ this.#state.stats.singleTokenReadbackCopyMs = 0;
432
786
  this.#state.stats.singleTokenOrchestrationMs = 0;
433
787
  this.#state.stats.pleHotVocabularyHits = 0;
434
788
  this.#state.stats.pleHotVocabularyMisses = 0;
@@ -452,6 +806,7 @@ export class PipelineGenerator {
452
806
  opts.topP = samplingConfig.topP;
453
807
  opts.topK = samplingConfig.topK;
454
808
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
809
+ opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
455
810
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
456
811
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
457
812
  const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
@@ -824,7 +1179,7 @@ export class PipelineGenerator {
824
1179
  // Generation Public API
825
1180
  // ==========================================================================
826
1181
 
827
- /**
1182
+ /*
828
1183
  * Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
829
1184
  * match. Intended for "prefix-reuse" workflows where a caller wants to run
830
1185
  * several decodes that share a common prompt prefix: prefill once with the
@@ -850,6 +1205,574 @@ export class PipelineGenerator {
850
1205
  this.#state.currentSeqLen = target;
851
1206
  }
852
1207
 
1208
+ async _createDiffusionGemmaSelfConditioningEmbeddings(canvasIds, selfConditioningLogits, opts) {
1209
+ const config = this.#state.modelConfig;
1210
+ const canvasLength = canvasIds.length;
1211
+ const hiddenSize = config.hiddenSize;
1212
+ const vocabSize = config.vocabSize;
1213
+ const elementCount = canvasLength * hiddenSize;
1214
+ const softEmbeddingState = normalizeSelfConditioningSoftEmbeddingState(
1215
+ selfConditioningLogits,
1216
+ canvasLength,
1217
+ hiddenSize
1218
+ );
1219
+ const logitsState = softEmbeddingState
1220
+ ? null
1221
+ : normalizeSelfConditioningLogitsState(selfConditioningLogits, canvasLength, vocabSize);
1222
+ const logits = (softEmbeddingState || logitsState)
1223
+ ? null
1224
+ : normalizeSelfConditioningLogits(selfConditioningLogits, canvasLength, vocabSize);
1225
+ const weights = this.#state.weights.get('diffusion_gemma_self_conditioning');
1226
+ if (!weights || typeof weights !== 'object') {
1227
+ throw new Error(
1228
+ 'DiffusionGemma self-conditioning weights were not loaded. ' +
1229
+ 'Expected model.decoder.self_conditioning tensors in the manifest.'
1230
+ );
1231
+ }
1232
+
1233
+ const embedBufferRaw = this.#state.weights.get('embed');
1234
+ if (!embedBufferRaw) {
1235
+ throw new Error('DiffusionGemma self-conditioning requires loaded embed_tokens weights.');
1236
+ }
1237
+ const borrowed = {
1238
+ preNorm: null,
1239
+ gateProj: null,
1240
+ upProj: null,
1241
+ downProj: null,
1242
+ softEmbedding: null,
1243
+ };
1244
+ let softLogitsTensor = null;
1245
+ let softLogitsOwned = true;
1246
+ let softEmbeddingStateTensor = null;
1247
+ let softEmbeddingStateOwned = false;
1248
+ let softmaxTensor = null;
1249
+ let softEmbeddings = null;
1250
+ let scaledSoftEmbeddings = null;
1251
+ let baseEmbeddings = null;
1252
+ let normedSoft = null;
1253
+ let gate = null;
1254
+ let up = null;
1255
+ let activated = null;
1256
+ let down = null;
1257
+ let combined = null;
1258
+ let output = null;
1259
+
1260
+ const releaseTensorOnce = (() => {
1261
+ const released = new Set();
1262
+ return (tensor) => {
1263
+ const buffer = tensor?.buffer ?? null;
1264
+ if (!buffer || released.has(buffer)) return;
1265
+ released.add(buffer);
1266
+ releaseBuffer(buffer);
1267
+ };
1268
+ })();
1269
+
1270
+ try {
1271
+ borrowed.preNorm = borrowNormWeight(weights.preNorm, 'diffusion_gemma_self_conditioning.pre_norm');
1272
+ borrowed.gateProj = borrowLinearWeight(weights.gateProj, 'diffusion_gemma_self_conditioning.gate_proj');
1273
+ borrowed.upProj = borrowLinearWeight(weights.upProj, 'diffusion_gemma_self_conditioning.up_proj');
1274
+ borrowed.downProj = borrowLinearWeight(weights.downProj, 'diffusion_gemma_self_conditioning.down_proj');
1275
+ if (isSplitWeightBuffer(embedBufferRaw) && this.#state.embeddingTranspose === true) {
1276
+ throw new Error(
1277
+ 'DiffusionGemma self-conditioning split embeddings require row-major embedding storage.'
1278
+ );
1279
+ }
1280
+ borrowed.softEmbedding = isSplitWeightBuffer(embedBufferRaw)
1281
+ ? null
1282
+ : borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1283
+
1284
+ const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1285
+ const embedDtype = isCpuWeightBuffer(embedBufferRaw)
1286
+ ? embedBufferRaw.dtype
1287
+ : getWeightDtype(embedBufferRaw);
1288
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
1289
+ baseEmbeddings = await embed(canvasIds, embedBuffer, {
1290
+ hiddenSize,
1291
+ vocabSize,
1292
+ scaleEmbeddings: config.scaleEmbeddings,
1293
+ debug: opts.debug,
1294
+ recorder: null,
1295
+ transpose: this.#state.embeddingTranspose,
1296
+ activationDtype: 'f32',
1297
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
1298
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1299
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1300
+ operatorDiagnostics: this.#state.operatorDiagnostics,
1301
+ });
1302
+
1303
+ if (softEmbeddingState) {
1304
+ softEmbeddingStateOwned = softEmbeddingState.releaseOnUse;
1305
+ softEmbeddings = createTensor(
1306
+ softEmbeddingState.buffer,
1307
+ softEmbeddingState.dtype,
1308
+ [canvasLength, hiddenSize],
1309
+ 'diffusion_gemma_self_conditioning_soft_embedding'
1310
+ );
1311
+ softEmbeddingStateTensor = softEmbeddings;
1312
+ if (!softEmbeddingState.scaled) {
1313
+ scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1314
+ count: elementCount,
1315
+ });
1316
+ if (softEmbeddingStateOwned) {
1317
+ releaseTensorOnce(softEmbeddings);
1318
+ }
1319
+ softEmbeddings = scaledSoftEmbeddings;
1320
+ softEmbeddingStateTensor = null;
1321
+ scaledSoftEmbeddings = null;
1322
+ }
1323
+ } else if (logitsState || logits) {
1324
+ const softmaxTemperature = logitsState?.temperature ?? 1.0;
1325
+ if (logitsState) {
1326
+ softLogitsOwned = logitsState.releaseOnUse;
1327
+ softLogitsTensor = createTensor(
1328
+ logitsState.logitsBuffer,
1329
+ logitsState.logitsDtype,
1330
+ [canvasLength, vocabSize],
1331
+ 'diffusion_gemma_self_conditioning_logits'
1332
+ );
1333
+ } else {
1334
+ const logitsBuffer = acquireBuffer(logits.byteLength, undefined, 'diffusion_gemma_self_conditioning_logits');
1335
+ uploadData(logitsBuffer, logits);
1336
+ softLogitsTensor = createTensor(logitsBuffer, 'f32', [canvasLength, vocabSize], 'diffusion_gemma_self_conditioning_logits');
1337
+ }
1338
+ if (canUseChunkedSoftEmbeddingLogits(
1339
+ logitsState,
1340
+ borrowed.softEmbedding?.value,
1341
+ this.#state.embeddingTranspose
1342
+ )) {
1343
+ softEmbeddings = await runSoftEmbeddingLogitsF16(
1344
+ softLogitsTensor,
1345
+ borrowed.softEmbedding.value,
1346
+ canvasLength,
1347
+ hiddenSize,
1348
+ vocabSize,
1349
+ {
1350
+ temperature: softmaxTemperature,
1351
+ chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1352
+ }
1353
+ );
1354
+ } else {
1355
+ softmaxTensor = await runSoftmax(softLogitsTensor, -1, {
1356
+ batchSize: canvasLength,
1357
+ size: vocabSize,
1358
+ temperature: softmaxTemperature,
1359
+ });
1360
+ softEmbeddings = isSplitWeightBuffer(embedBufferRaw)
1361
+ ? await runSoftEmbeddingSplitF16(
1362
+ softmaxTensor,
1363
+ embedBufferRaw,
1364
+ canvasLength,
1365
+ hiddenSize,
1366
+ vocabSize
1367
+ )
1368
+ : await runMatmul(
1369
+ softmaxTensor,
1370
+ borrowed.softEmbedding.value,
1371
+ canvasLength,
1372
+ hiddenSize,
1373
+ vocabSize,
1374
+ {
1375
+ transposeB: this.#state.embeddingTranspose === true,
1376
+ role: 'diffusion_gemma_self_conditioning_embed',
1377
+ outputDtype: 'f32',
1378
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1379
+ }
1380
+ );
1381
+ }
1382
+ scaledSoftEmbeddings = await runScale(softEmbeddings, Math.sqrt(hiddenSize), {
1383
+ count: elementCount,
1384
+ });
1385
+ if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1386
+ releaseTensorOnce(softEmbeddings);
1387
+ }
1388
+ softEmbeddings = scaledSoftEmbeddings;
1389
+ scaledSoftEmbeddings = null;
1390
+ } else {
1391
+ const zeroBuffer = acquireBuffer(elementCount * Float32Array.BYTES_PER_ELEMENT, undefined, 'diffusion_gemma_self_conditioning_zero');
1392
+ uploadData(zeroBuffer, new Uint8Array(elementCount * Float32Array.BYTES_PER_ELEMENT));
1393
+ softEmbeddings = createTensor(zeroBuffer, 'f32', [canvasLength, hiddenSize], 'diffusion_gemma_self_conditioning_zero');
1394
+ }
1395
+
1396
+ normedSoft = await runRMSNorm(softEmbeddings, borrowed.preNorm.value, config.rmsNormEps, {
1397
+ batchSize: canvasLength,
1398
+ hiddenSize,
1399
+ rmsNormWeightOffset: false,
1400
+ });
1401
+ const intermediateSize = config.intermediateSize;
1402
+ gate = await runMatmul(
1403
+ normedSoft,
1404
+ borrowed.gateProj.value,
1405
+ canvasLength,
1406
+ intermediateSize,
1407
+ hiddenSize,
1408
+ {
1409
+ transposeB: 'auto',
1410
+ role: 'diffusion_gemma_self_conditioning_gate',
1411
+ outputDtype: 'f32',
1412
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1413
+ }
1414
+ );
1415
+ up = await runMatmul(
1416
+ normedSoft,
1417
+ borrowed.upProj.value,
1418
+ canvasLength,
1419
+ intermediateSize,
1420
+ hiddenSize,
1421
+ {
1422
+ transposeB: 'auto',
1423
+ role: 'diffusion_gemma_self_conditioning_up',
1424
+ outputDtype: 'f32',
1425
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1426
+ }
1427
+ );
1428
+ activated = await runGeLU(up, {
1429
+ size: canvasLength * intermediateSize,
1430
+ gate,
1431
+ });
1432
+ down = await runMatmul(
1433
+ activated,
1434
+ borrowed.downProj.value,
1435
+ canvasLength,
1436
+ hiddenSize,
1437
+ intermediateSize,
1438
+ {
1439
+ transposeB: 'auto',
1440
+ role: 'diffusion_gemma_self_conditioning_down',
1441
+ outputDtype: 'f32',
1442
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1443
+ }
1444
+ );
1445
+ combined = await runResidualAdd(baseEmbeddings, down, elementCount, {
1446
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
1447
+ });
1448
+ const postNormWeight = weights.postNorm
1449
+ ? borrowNormWeight(weights.postNorm, 'diffusion_gemma_self_conditioning.post_norm')
1450
+ : {
1451
+ value: getQKNormOnesBuffer(hiddenSize),
1452
+ owned: false,
1453
+ };
1454
+ borrowed.postNorm = postNormWeight;
1455
+ output = await runRMSNorm(combined, postNormWeight.value, config.rmsNormEps, {
1456
+ batchSize: canvasLength,
1457
+ hiddenSize,
1458
+ rmsNormWeightOffset: false,
1459
+ });
1460
+ return output;
1461
+ } catch (error) {
1462
+ releaseTensorOnce(output);
1463
+ throw error;
1464
+ } finally {
1465
+ releaseTensorOnce(combined);
1466
+ releaseTensorOnce(down);
1467
+ releaseTensorOnce(activated);
1468
+ releaseTensorOnce(up);
1469
+ releaseTensorOnce(gate);
1470
+ releaseTensorOnce(normedSoft);
1471
+ if (softEmbeddings !== softEmbeddingStateTensor || softEmbeddingStateOwned) {
1472
+ releaseTensorOnce(softEmbeddings);
1473
+ }
1474
+ releaseTensorOnce(softmaxTensor);
1475
+ if (softLogitsOwned) {
1476
+ releaseTensorOnce(softLogitsTensor);
1477
+ }
1478
+ releaseTensorOnce(baseEmbeddings);
1479
+ releaseBorrowedWeight(borrowed.postNorm);
1480
+ releaseBorrowedWeight(borrowed.downProj);
1481
+ releaseBorrowedWeight(borrowed.upProj);
1482
+ releaseBorrowedWeight(borrowed.gateProj);
1483
+ releaseBorrowedWeight(borrowed.preNorm);
1484
+ releaseBorrowedWeight(borrowed.softEmbedding);
1485
+ }
1486
+ }
1487
+
1488
+ async _createDiffusionGemmaSelfConditioningSoftEmbeddingState(logitsState, canvasLength, hiddenSize, vocabSize) {
1489
+ const embedBufferRaw = this.#state.weights.get('embed');
1490
+ if (!canUseChunkedSoftEmbeddingLogits(
1491
+ logitsState,
1492
+ embedBufferRaw,
1493
+ this.#state.embeddingTranspose
1494
+ )) {
1495
+ return null;
1496
+ }
1497
+
1498
+ const borrowed = borrowLinearWeight(embedBufferRaw, 'diffusion_gemma_self_conditioning.embed_tokens');
1499
+ let softEmbedding = null;
1500
+ let scaledSoftEmbedding = null;
1501
+ try {
1502
+ const logitsTensor = createTensor(
1503
+ logitsState.logitsBuffer,
1504
+ logitsState.logitsDtype,
1505
+ [canvasLength, vocabSize],
1506
+ 'diffusion_gemma_self_conditioning_logits'
1507
+ );
1508
+ softEmbedding = await runSoftEmbeddingLogitsF16(
1509
+ logitsTensor,
1510
+ borrowed.value,
1511
+ canvasLength,
1512
+ hiddenSize,
1513
+ vocabSize,
1514
+ {
1515
+ temperature: logitsState.temperature,
1516
+ chunkRows: resolveDiffusionGemmaSoftEmbeddingChunkRows(this.#state.runtimeConfig),
1517
+ }
1518
+ );
1519
+ scaledSoftEmbedding = await runScale(softEmbedding, Math.sqrt(hiddenSize), {
1520
+ count: canvasLength * hiddenSize,
1521
+ });
1522
+ const returnedBuffer = scaledSoftEmbedding.buffer;
1523
+ return {
1524
+ kind: 'soft_embedding',
1525
+ buffer: returnedBuffer,
1526
+ dtype: scaledSoftEmbedding.dtype,
1527
+ canvasLength,
1528
+ hiddenSize,
1529
+ scaled: true,
1530
+ releaseOnUse: true,
1531
+ release() {
1532
+ releaseBuffer(returnedBuffer);
1533
+ },
1534
+ };
1535
+ } finally {
1536
+ if (softEmbedding?.buffer && softEmbedding.buffer !== scaledSoftEmbedding?.buffer) {
1537
+ releaseBuffer(softEmbedding.buffer);
1538
+ }
1539
+ releaseBorrowedWeight(borrowed);
1540
+ }
1541
+ }
1542
+
1543
+ async computeDiffusionGemmaCanvasLogits(args, options = {}) {
1544
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
1545
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
1546
+ throw new Error('Generation already in progress');
1547
+ }
1548
+ const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasLogits');
1549
+ if (canvasIds.length === 0) {
1550
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasLogits requires at least one canvas token.');
1551
+ }
1552
+ this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasLogits.canvas');
1553
+ const seqLenBefore = this.#state.currentSeqLen;
1554
+ const opts = {
1555
+ ...resolvePrefillOptions(this.#state, {
1556
+ ...options,
1557
+ useChatTemplate: false,
1558
+ }),
1559
+ _diffusionGemmaDecoder: true,
1560
+ };
1561
+ let selfConditioned = null;
1562
+ let currentHiddenBuffer = null;
1563
+ try {
1564
+ selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1565
+ canvasIds,
1566
+ args?.selfConditioningLogits ?? null,
1567
+ opts
1568
+ );
1569
+ const prefillResult = await this._prefillToHidden(canvasIds, {
1570
+ ...opts,
1571
+ embeddingOverrides: {
1572
+ offset: 0,
1573
+ prefixLength: canvasIds.length,
1574
+ embeddings: selfConditioned.buffer,
1575
+ },
1576
+ });
1577
+ const {
1578
+ numTokens,
1579
+ currentRecorder,
1580
+ recordProfile,
1581
+ debugCheckBuffer,
1582
+ } = prefillResult;
1583
+ currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1584
+ if (currentRecorder) {
1585
+ await currentRecorder.submitAndWait();
1586
+ await recordProfile(currentRecorder);
1587
+ } else {
1588
+ const device = getDevice();
1589
+ if (device) {
1590
+ await device.queue.onSubmittedWorkDone();
1591
+ }
1592
+ }
1593
+ const logits = await computeLogits(
1594
+ currentHiddenBuffer,
1595
+ numTokens,
1596
+ getLogitsWeights(this.#state),
1597
+ getLogitsConfig(this.#state),
1598
+ this.#state.useGPU,
1599
+ this.#state.debugFlags,
1600
+ undefined,
1601
+ debugCheckBuffer,
1602
+ this.#state.runtimeConfig.shared.debug.probes,
1603
+ { lastPositionOnly: false },
1604
+ this.#state.operatorDiagnostics
1605
+ );
1606
+ const expected = canvasIds.length * this.#state.modelConfig.vocabSize;
1607
+ if (logits.length !== expected) {
1608
+ throw new Error(
1609
+ `[DiffusionGemma] canvas logits length mismatch: expected ${expected}, got ${logits.length}.`
1610
+ );
1611
+ }
1612
+ return logits;
1613
+ } finally {
1614
+ this.#state.currentSeqLen = seqLenBefore;
1615
+ if (currentHiddenBuffer) {
1616
+ releaseBuffer(currentHiddenBuffer);
1617
+ }
1618
+ if (selfConditioned?.buffer) {
1619
+ releaseBuffer(selfConditioned.buffer);
1620
+ }
1621
+ }
1622
+ }
1623
+
1624
+ async computeDiffusionGemmaCanvasStep(args, options = {}) {
1625
+ if (!this.#state.isLoaded) throw new Error('Model not loaded');
1626
+ if (this.#state.isGenerating && options.__internalGenerate !== true) {
1627
+ throw new Error('Generation already in progress');
1628
+ }
1629
+ const canvasIds = normalizeCanvasTokenIds(args?.canvas, 'computeDiffusionGemmaCanvasStep');
1630
+ if (canvasIds.length === 0) {
1631
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires at least one canvas token.');
1632
+ }
1633
+ this._assertTokenIdsInRange(canvasIds, 'computeDiffusionGemmaCanvasStep.canvas');
1634
+ const temperature = args?.temperature;
1635
+ if (typeof temperature !== 'number' || !Number.isFinite(temperature) || temperature <= 0) {
1636
+ throw new Error('[DiffusionGemma] computeDiffusionGemmaCanvasStep requires a positive temperature.');
1637
+ }
1638
+ const seqLenBefore = this.#state.currentSeqLen;
1639
+ const opts = {
1640
+ ...resolvePrefillOptions(this.#state, {
1641
+ ...options,
1642
+ useChatTemplate: false,
1643
+ }),
1644
+ _diffusionGemmaDecoder: true,
1645
+ };
1646
+ let selfConditioned = null;
1647
+ let currentHiddenBuffer = null;
1648
+ let logitsBuffer = null;
1649
+ let statsBuffers = null;
1650
+ try {
1651
+ selfConditioned = await this._createDiffusionGemmaSelfConditioningEmbeddings(
1652
+ canvasIds,
1653
+ args?.selfConditioningLogits ?? null,
1654
+ opts
1655
+ );
1656
+ const prefillResult = await this._prefillToHidden(canvasIds, {
1657
+ ...opts,
1658
+ embeddingOverrides: {
1659
+ offset: 0,
1660
+ prefixLength: canvasIds.length,
1661
+ embeddings: selfConditioned.buffer,
1662
+ },
1663
+ });
1664
+ const {
1665
+ numTokens,
1666
+ currentRecorder,
1667
+ recordProfile,
1668
+ debugCheckBuffer,
1669
+ } = prefillResult;
1670
+ currentHiddenBuffer = prefillResult.currentHiddenBuffer;
1671
+ if (currentRecorder) {
1672
+ await currentRecorder.submitAndWait();
1673
+ await recordProfile(currentRecorder);
1674
+ } else {
1675
+ const device = getDevice();
1676
+ if (device) {
1677
+ await device.queue.onSubmittedWorkDone();
1678
+ }
1679
+ }
1680
+ if (selfConditioned?.buffer) {
1681
+ releaseBuffer(selfConditioned.buffer);
1682
+ selfConditioned = null;
1683
+ }
1684
+
1685
+ const logitsResult = await computeLogitsGPU(
1686
+ currentHiddenBuffer,
1687
+ numTokens,
1688
+ getLogitsWeights(this.#state),
1689
+ getLogitsConfig(this.#state),
1690
+ this.#state.debugFlags,
1691
+ this.#state.operatorDiagnostics
1692
+ );
1693
+ if (!logitsResult?.logitsBuffer) {
1694
+ throw new Error('[DiffusionGemma] GPU canvas step requires GPU logits.');
1695
+ }
1696
+ logitsBuffer = logitsResult.logitsBuffer;
1697
+ if (currentHiddenBuffer) {
1698
+ releaseBuffer(currentHiddenBuffer);
1699
+ currentHiddenBuffer = null;
1700
+ }
1701
+ if (logitsResult.logitsDtype !== 'f32') {
1702
+ throw new Error(
1703
+ `[DiffusionGemma] GPU canvas stats require f32 logits, got "${logitsResult.logitsDtype}".`
1704
+ );
1705
+ }
1706
+ if (logitsResult.vocabSize !== this.#state.modelConfig.vocabSize) {
1707
+ throw new Error(
1708
+ `[DiffusionGemma] canvas logits vocab mismatch: expected ${this.#state.modelConfig.vocabSize}, ` +
1709
+ `got ${logitsResult.vocabSize}.`
1710
+ );
1711
+ }
1712
+
1713
+ statsBuffers = await runDiffusionGemmaCanvasStats(logitsBuffer, {
1714
+ canvasLength: canvasIds.length,
1715
+ vocabSize: logitsResult.vocabSize,
1716
+ temperature,
1717
+ padTokenId: this.#state.modelConfig.diffusionGemma?.padTokenId ?? null,
1718
+ logitSoftcap: this.#state.modelConfig.finalLogitSoftcapping ?? 0,
1719
+ });
1720
+ const [argmaxData, entropyData] = await Promise.all([
1721
+ readBuffer(statsBuffers.argmaxBuffer, canvasIds.length * Uint32Array.BYTES_PER_ELEMENT),
1722
+ readBuffer(statsBuffers.entropyBuffer, canvasIds.length * Float32Array.BYTES_PER_ELEMENT),
1723
+ ]);
1724
+ const argmaxCanvas = Int32Array.from(new Uint32Array(argmaxData));
1725
+ const entropies = new Float32Array(entropyData);
1726
+ const logitsState = {
1727
+ logitsBuffer,
1728
+ logitsDtype: logitsResult.logitsDtype,
1729
+ vocabSize: logitsResult.vocabSize,
1730
+ canvasLength: canvasIds.length,
1731
+ temperature,
1732
+ releaseOnUse: true,
1733
+ };
1734
+ const selfConditioningState =
1735
+ await this._createDiffusionGemmaSelfConditioningSoftEmbeddingState(
1736
+ logitsState,
1737
+ canvasIds.length,
1738
+ this.#state.modelConfig.hiddenSize,
1739
+ logitsResult.vocabSize
1740
+ );
1741
+ if (selfConditioningState) {
1742
+ releaseBuffer(logitsBuffer);
1743
+ logitsBuffer = null;
1744
+ } else {
1745
+ const returnedLogitsBuffer = logitsBuffer;
1746
+ logitsBuffer = null;
1747
+ logitsState.release = () => {
1748
+ releaseBuffer(returnedLogitsBuffer);
1749
+ };
1750
+ }
1751
+ return {
1752
+ argmaxCanvas,
1753
+ entropies,
1754
+ selfConditioningLogits: selfConditioningState ?? logitsState,
1755
+ };
1756
+ } finally {
1757
+ this.#state.currentSeqLen = seqLenBefore;
1758
+ if (statsBuffers?.argmaxBuffer) {
1759
+ releaseBuffer(statsBuffers.argmaxBuffer);
1760
+ }
1761
+ if (statsBuffers?.entropyBuffer) {
1762
+ releaseBuffer(statsBuffers.entropyBuffer);
1763
+ }
1764
+ if (logitsBuffer) {
1765
+ releaseBuffer(logitsBuffer);
1766
+ }
1767
+ if (currentHiddenBuffer) {
1768
+ releaseBuffer(currentHiddenBuffer);
1769
+ }
1770
+ if (selfConditioned?.buffer) {
1771
+ releaseBuffer(selfConditioned.buffer);
1772
+ }
1773
+ }
1774
+ }
1775
+
853
1776
  async *generate(prompt, options = {}) {
854
1777
  yield* this._generateTokensInternal(prompt, options, 'text');
855
1778
  }
@@ -869,12 +1792,21 @@ export class PipelineGenerator {
869
1792
  this.#state.stats.gpuTimePrefillMs = undefined;
870
1793
  this.#state.stats.gpuTimeDecodeMs = undefined;
871
1794
  this.#state.stats.decodeRecordMs = 0;
1795
+ this.#state.stats.decodeRecordOps = 0;
1796
+ this.#state.stats.decodeRecordPasses = 0;
1797
+ this.#state.stats.decodeRecordOpLabels = {};
872
1798
  this.#state.stats.decodeSubmitWaitMs = 0;
873
1799
  this.#state.stats.decodeReadbackWaitMs = 0;
1800
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
1801
+ this.#state.stats.decodeReadbackCleanupMs = 0;
1802
+ this.#state.stats.decodeReadbackCopyMs = 0;
874
1803
  this.#state.stats.prefillRecordMs = 0;
875
1804
  this.#state.stats.prefillSubmitWaitMs = 0;
876
1805
  this.#state.stats.singleTokenSubmitWaitMs = 0;
877
1806
  this.#state.stats.singleTokenReadbackWaitMs = 0;
1807
+ this.#state.stats.singleTokenReadbackMapWaitMs = 0;
1808
+ this.#state.stats.singleTokenReadbackCleanupMs = 0;
1809
+ this.#state.stats.singleTokenReadbackCopyMs = 0;
878
1810
  this.#state.stats.singleTokenOrchestrationMs = 0;
879
1811
  this.#state.stats.pleHotVocabularyHits = 0;
880
1812
  this.#state.stats.pleHotVocabularyMisses = 0;
@@ -896,6 +1828,7 @@ export class PipelineGenerator {
896
1828
  opts.topP = samplingConfig.topP;
897
1829
  opts.topK = samplingConfig.topK;
898
1830
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
1831
+ opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this.#state, samplingConfig);
899
1832
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
900
1833
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
901
1834
  const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
@@ -1262,8 +2195,14 @@ export class PipelineGenerator {
1262
2195
  this.#state.stats.prefillProfileSteps = [];
1263
2196
  this.#state.decodeRing?.reset();
1264
2197
  this.#state.stats.decodeRecordMs = 0;
2198
+ this.#state.stats.decodeRecordOps = 0;
2199
+ this.#state.stats.decodeRecordPasses = 0;
2200
+ this.#state.stats.decodeRecordOpLabels = {};
1265
2201
  this.#state.stats.decodeSubmitWaitMs = 0;
1266
2202
  this.#state.stats.decodeReadbackWaitMs = 0;
2203
+ this.#state.stats.decodeReadbackMapWaitMs = 0;
2204
+ this.#state.stats.decodeReadbackCleanupMs = 0;
2205
+ this.#state.stats.decodeReadbackCopyMs = 0;
1267
2206
  this.#state.stats.ttftMs = 0;
1268
2207
  const startTime = performance.now();
1269
2208
 
@@ -1290,6 +2229,7 @@ export class PipelineGenerator {
1290
2229
  topK: opts.topK,
1291
2230
  padTokenId,
1292
2231
  seed: opts.seed,
2232
+ suppressTokenIds: opts.suppressTokenIds,
1293
2233
  });
1294
2234
 
1295
2235
  const decodeRuntime = {
@@ -1372,6 +2312,7 @@ export class PipelineGenerator {
1372
2312
  const hasLinearLayers = hasLinearAttentionLayers(this.#state.modelConfig.layerTypes);
1373
2313
  const replayPrefillDecode = usesReplayPrefillDecode(this.#state);
1374
2314
  const gpuSamplingAvailable = isGPUSamplingAvailable() && !hasCpuWeights;
2315
+ const hasSuppressedSamplingTokens = Array.isArray(opts.suppressTokenIds) && opts.suppressTokenIds.length > 0;
1375
2316
  const hasRangeBackedPerLayerInputs = hasRangeBackedPerLayerInputEmbeddings({
1376
2317
  config: this.#state.modelConfig,
1377
2318
  weights: this.#state.weights,
@@ -1379,6 +2320,7 @@ export class PipelineGenerator {
1379
2320
  const hasGpuSplitPerLayerInputs = hasGpuSplitPerLayerInputEmbeddings({
1380
2321
  config: this.#state.modelConfig,
1381
2322
  weights: this.#state.weights,
2323
+ perLayerInputsSession: resolvedPerLayerInputsSession,
1382
2324
  });
1383
2325
  const pleHotVocabularyRuntime = getPleHotVocabularyRuntime({ weights: this.#state.weights });
1384
2326
  const resolveCurrentHotVocabularyBatchDecodeAvailable = () => resolveHotVocabularyBatchDecodeAvailability({
@@ -1394,7 +2336,7 @@ export class PipelineGenerator {
1394
2336
  batchSize: executionPlan.batchSize,
1395
2337
  useGPU: this.#state.useGPU,
1396
2338
  gpuSamplingAvailable,
1397
- disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
2339
+ disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
1398
2340
  disableCommandBatching: executionPlan.disableCommandBatching,
1399
2341
  isBdpaPagedLayout: this.#state.kvCache?.layout === 'bdpa_paged',
1400
2342
  finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
@@ -1409,6 +2351,7 @@ export class PipelineGenerator {
1409
2351
  else if (!this.#state.useGPU) reason = 'no_gpu';
1410
2352
  else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
1411
2353
  else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
2354
+ else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
1412
2355
  else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
1413
2356
  else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
1414
2357
  else if (this.#state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
@@ -1454,21 +2397,58 @@ export class PipelineGenerator {
1454
2397
  hasHotVocabularyBatchDecode: hotVocabularyBatchDecodeAvailable,
1455
2398
  hasGpuSplitPerLayerInputs,
1456
2399
  hasLinearAttentionLayers: hasLinearLayers,
2400
+ modelId: this.#state.modelConfig.modelId ?? executionPlan.kernelPathId,
2401
+ activationDtype: executionPlan.activationDtype,
2402
+ currentSeqLen: this.#state.currentSeqLen,
2403
+ maxDecodeTokens: opts.maxTokens,
2404
+ numLayers: this.#state.modelConfig.numLayers,
2405
+ hiddenSize: this.#state.modelConfig.hiddenSize,
1457
2406
  });
1458
2407
  const requestedBatchTokens = executionPlan.batchSize * intervalBatches;
1459
2408
  const boundedBatchTokens = maxBatchDecodeTokens == null
1460
2409
  ? requestedBatchTokens
1461
2410
  : Math.min(requestedBatchTokens, maxBatchDecodeTokens);
1462
2411
  const thisBatchSize = Math.min(boundedBatchTokens, remaining);
2412
+ this.#state.batchingStats.requestedBatchTokens = Math.max(
2413
+ this.#state.batchingStats.requestedBatchTokens ?? 0,
2414
+ requestedBatchTokens
2415
+ );
2416
+ this.#state.batchingStats.effectiveBatchTokens = Math.max(
2417
+ this.#state.batchingStats.effectiveBatchTokens ?? 0,
2418
+ thisBatchSize
2419
+ );
2420
+ if (maxBatchDecodeTokens != null) {
2421
+ this.#state.batchingStats.maxBatchTokenCap = Math.max(
2422
+ this.#state.batchingStats.maxBatchTokenCap ?? 0,
2423
+ maxBatchDecodeTokens
2424
+ );
2425
+ }
2426
+ if (thisBatchSize < requestedBatchTokens) {
2427
+ this.#state.batchingStats.batchClampCount = (this.#state.batchingStats.batchClampCount ?? 0) + 1;
2428
+ }
1463
2429
  const lastToken = generatedIds[generatedIds.length - 1];
2430
+ const boundedExecutionPlan = thisBatchSize < requestedBatchTokens
2431
+ ? {
2432
+ ...executionPlan,
2433
+ batchSize: Math.min(executionPlan.batchSize, thisBatchSize),
2434
+ readbackInterval: readbackInterval == null
2435
+ ? null
2436
+ : Math.min(intervalBatches, thisBatchSize),
2437
+ }
2438
+ : executionPlan;
2439
+ const batchOpts = boundedExecutionPlan === executionPlan
2440
+ ? opts
2441
+ : { ...opts, executionPlan: boundedExecutionPlan };
1464
2442
 
1465
2443
  try {
1466
- const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, opts);
2444
+ const batchResult = await this._generateNTokensGPU(lastToken, thisBatchSize, generatedIds, batchOpts);
1467
2445
  let batchTokens = [];
1468
2446
  let hitStop = false;
2447
+ let stopTokenId = null;
1469
2448
  for (const tokenId of batchResult.tokens) {
1470
2449
  if (isStopToken(tokenId, stopTokenIds, eosToken)) {
1471
2450
  hitStop = true;
2451
+ stopTokenId = tokenId;
1472
2452
  break;
1473
2453
  }
1474
2454
  generatedIds.push(tokenId);
@@ -1497,7 +2477,7 @@ export class PipelineGenerator {
1497
2477
  }
1498
2478
  }
1499
2479
  if (hitStop) {
1500
- this._recordStopReason('stop-token');
2480
+ this._recordStopReason('stop-token', stopTokenId);
1501
2481
  break;
1502
2482
  }
1503
2483
  if (shouldDisableBatchDecodeAfterShortBatch({
@@ -1721,6 +2701,7 @@ export class PipelineGenerator {
1721
2701
  this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
1722
2702
  this.#state.stats.tokensGenerated = tokensGenerated;
1723
2703
  this.#state.stats.decodeTokens = tokensGenerated;
2704
+ this.#state.stats.batching = { ...this.#state.batchingStats };
1724
2705
  }
1725
2706
 
1726
2707
  async _prefillToHidden(inputIds, opts) {
@@ -1733,6 +2714,8 @@ export class PipelineGenerator {
1733
2714
  const numTokens = inputIds.length;
1734
2715
  const config = this.#state.modelConfig;
1735
2716
  const startPos = this.#state.currentSeqLen;
2717
+ const tracePrefillEnabled = isTraceEnabled('perf');
2718
+ const prefillTraceStart = tracePrefillEnabled ? performance.now() : 0;
1736
2719
  const returnHidden = opts?._returnHidden === true;
1737
2720
  const embeddingInputIds = resolvePrefillEmbeddingInputIds(
1738
2721
  inputIds,
@@ -1768,13 +2751,14 @@ export class PipelineGenerator {
1768
2751
  }
1769
2752
 
1770
2753
  const embedBufferRaw = this.#state.weights.get('embed');
1771
- if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
2754
+ if (!isGpuBufferInstance(embedBufferRaw) && !isWeightBuffer(embedBufferRaw) && !isCpuWeightBuffer(embedBufferRaw) && !isSplitWeightBuffer(embedBufferRaw) && !(embedBufferRaw instanceof Float32Array)) {
1772
2755
  throw new Error('Embed buffer not found or not a supported buffer type');
1773
2756
  }
1774
2757
  const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
1775
2758
  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
1776
2759
  ? embedBufferRaw.dtype
1777
2760
  : getWeightDtype(embedBufferRaw);
2761
+ const embedMetadata = getWeightMetadata(embedBufferRaw);
1778
2762
  if (opts.debug) {
1779
2763
  const embedSize = isGpuBufferInstance(embedBuffer) ? embedBuffer.size : 'N/A';
1780
2764
  log.debug('Pipeline', `Embed buffer: type=${embedBuffer?.constructor?.name}, size=${embedSize}, dtype=${embedDtype}`);
@@ -1789,7 +2773,9 @@ export class PipelineGenerator {
1789
2773
  );
1790
2774
  const createRecorder = (label) => {
1791
2775
  if (!device || disableCommandBatching) return undefined;
1792
- return opts.profile ? createProfilingRecorder(label) : createCommandRecorder(label);
2776
+ return opts.profile
2777
+ ? createProfilingRecorder(label, device)
2778
+ : createCommandRecorder(label, { recordLabels: opts.debug === true }, device);
1793
2779
  };
1794
2780
  const recorder = createRecorder('prefill');
1795
2781
  const debugCheckBuffer = this.#state.debug
@@ -1805,6 +2791,7 @@ export class PipelineGenerator {
1805
2791
  opts.executionPlan
1806
2792
  );
1807
2793
  context.currentTokenIds = inputIds;
2794
+ context.diffusionGemmaDecoder = opts?._diffusionGemmaDecoder === true;
1808
2795
  context.multimodalBidirectionalSpan = multimodalBidirectionalSpan == null
1809
2796
  ? null
1810
2797
  : {
@@ -1854,6 +2841,16 @@ export class PipelineGenerator {
1854
2841
 
1855
2842
  const activationDtype = opts.executionPlan?.activationDtype ?? this._getEffectiveActivationDtype();
1856
2843
  const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
2844
+ if (tracePrefillEnabled) {
2845
+ trace.perf('Prefill phase start', {
2846
+ numTokens,
2847
+ startPos,
2848
+ numLayers: config.numLayers,
2849
+ activationDtype,
2850
+ returnHidden,
2851
+ });
2852
+ }
2853
+ const embedTraceStart = tracePrefillEnabled ? performance.now() : 0;
1857
2854
  let baseEmbeddings = await embed(embeddingInputIds, embedBuffer, {
1858
2855
  hiddenSize: config.hiddenSize,
1859
2856
  vocabSize: config.vocabSize,
@@ -1864,18 +2861,29 @@ export class PipelineGenerator {
1864
2861
  debugProbes: this.#state.runtimeConfig.shared.debug.probes,
1865
2862
  operatorDiagnostics: this.#state.operatorDiagnostics,
1866
2863
  activationDtype,
1867
- embeddingDtype: selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', { dtype: embedDtype }),
2864
+ embeddingDtype: selectRuleValue('inference', 'dtype', 'embeddingDtype', { dtype: embedDtype }),
2865
+ embeddingStorageEncoding: embedMetadata?.storageEncoding ?? null,
1868
2866
  executionPolicies: this.#state.executionV1State?.policies ?? null,
1869
2867
  });
2868
+ if (tracePrefillEnabled) {
2869
+ trace.perf('Prefill embed complete', {
2870
+ numTokens,
2871
+ elapsedMs: performance.now() - embedTraceStart,
2872
+ });
2873
+ }
1870
2874
  let hiddenStates = baseEmbeddings;
1871
2875
  let perLayerInputs = null;
2876
+ const perLayerInputsTraceStart = tracePrefillEnabled ? performance.now() : 0;
1872
2877
  try {
1873
2878
  hiddenStates = await applyPrefixEmbeddingOverride(
1874
2879
  baseEmbeddings,
1875
2880
  embeddingOverride,
1876
2881
  config.hiddenSize,
1877
2882
  '_prefill',
1878
- this.#state.executionV1State?.policies ?? null
2883
+ {
2884
+ executionPolicies: this.#state.executionV1State?.policies ?? null,
2885
+ transitionDeclaredBy: resolvePrefixEmbeddingOverrideTransitionDeclaredBy(this.#state.executionV1State),
2886
+ }
1879
2887
  );
1880
2888
  perLayerInputs = await preparePerLayerInputs(
1881
2889
  embeddingInputIds,
@@ -1886,6 +2894,13 @@ export class PipelineGenerator {
1886
2894
  pleCache: this.#state.pleCache ?? null,
1887
2895
  }
1888
2896
  );
2897
+ if (tracePrefillEnabled) {
2898
+ trace.perf('Prefill per-layer inputs complete', {
2899
+ numTokens,
2900
+ elapsedMs: performance.now() - perLayerInputsTraceStart,
2901
+ materialized: Array.isArray(perLayerInputs),
2902
+ });
2903
+ }
1889
2904
  } catch (error) {
1890
2905
  if (isGpuBufferInstance(hiddenStates?.buffer)) {
1891
2906
  releaseBuffer(hiddenStates.buffer);
@@ -1934,7 +2949,7 @@ export class PipelineGenerator {
1934
2949
  if (this.#state.finitenessBuffer) {
1935
2950
  const device = getDevice();
1936
2951
  if (device) {
1937
- device.queue.writeBuffer(this.#state.finitenessBuffer, 0, new Uint32Array([0, 0, 0, 0]));
2952
+ device.queue.writeBuffer(this.#state.finitenessBuffer, 0, FINITENESS_RESET_WORDS);
1938
2953
  }
1939
2954
  }
1940
2955
 
@@ -1951,6 +2966,8 @@ export class PipelineGenerator {
1951
2966
  let currentHiddenBuffer = hiddenStates.buffer;
1952
2967
  let prefillRecordMs = 0;
1953
2968
  let prefillSubmitWaitMs = 0;
2969
+ const layerLoopTraceStart = tracePrefillEnabled ? performance.now() : 0;
2970
+ const traceLayerHealthEnabled = isTraceEnabled('logits');
1954
2971
  try {
1955
2972
  for (let l = 0; l < config.numLayers; l++) {
1956
2973
  // Per-layer hard cancellation: when the caller's AbortSignal aborts,
@@ -1987,6 +3004,7 @@ export class PipelineGenerator {
1987
3004
 
1988
3005
  const isCheckpoint = useCheckpoints && opts.debugLayers?.includes(l);
1989
3006
  const isChunkBoundary = !isCheckpoint
3007
+ && !traceLayerHealthEnabled
1990
3008
  && currentRecorder
1991
3009
  && l < config.numLayers - 1
1992
3010
  && (l + 1) % prefillRecorderChunkLayers === 0;
@@ -2002,6 +3020,27 @@ export class PipelineGenerator {
2002
3020
  currentRecorder = undefined;
2003
3021
  }
2004
3022
 
3023
+ if (traceLayerHealthEnabled && currentRecorder) {
3024
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
3025
+ currentHiddenBuffer,
3026
+ currentRecorder,
3027
+ 'prefill_trace_layer_health_carry'
3028
+ );
3029
+ const traceSubmitStart = performance.now();
3030
+ await currentRecorder.submitAndWait();
3031
+ await recordProfile(currentRecorder);
3032
+ prefillSubmitWaitMs += performance.now() - traceSubmitStart;
3033
+ await traceActivationHealth(
3034
+ `PREFILL_LAYER_${l}_HEALTH`,
3035
+ currentHiddenBuffer,
3036
+ activationDtype,
3037
+ numTokens * config.hiddenSize
3038
+ );
3039
+ currentRecorder = l < config.numLayers - 1
3040
+ ? createRecorder('prefill-trace')
3041
+ : undefined;
3042
+ }
3043
+
2005
3044
  const shouldDebug = opts.debug && currentHiddenBuffer && (!recorder || isCheckpoint);
2006
3045
  if (shouldDebug && !currentRecorder) {
2007
3046
  const device = getDevice();
@@ -2056,10 +3095,10 @@ export class PipelineGenerator {
2056
3095
  // tracked buffers when GPU work completes. Profile runs keep the
2057
3096
  // sync path because resolveProfileTimings requires the submitted
2058
3097
  // work to be done.
2059
- const manifestSubmit = this.#state.modelConfig?.sessionSettings?.prefillChunkSubmitMode;
2060
- const chunkSubmitMode = (manifestSubmit !== null && manifestSubmit !== undefined)
2061
- ? manifestSubmit
2062
- : (this.#state.runtimeConfig?.inference?.session?.prefillChunkSubmitMode ?? 'sync');
3098
+ const chunkSubmitMode = resolvePrefillChunkSubmitMode(
3099
+ this.#state.runtimeConfig,
3100
+ this.#state.modelConfig
3101
+ );
2063
3102
  const chunkSubmitStart = performance.now();
2064
3103
  if (chunkSubmitMode === 'async' && !opts.profile) {
2065
3104
  currentRecorder.submit({ cleanup: 'deferred' });
@@ -2068,9 +3107,31 @@ export class PipelineGenerator {
2068
3107
  await recordProfile(currentRecorder);
2069
3108
  }
2070
3109
  prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
3110
+ await traceActivationHealth(
3111
+ `PREFILL_LAYER_${l}_HEALTH`,
3112
+ currentHiddenBuffer,
3113
+ activationDtype,
3114
+ numTokens * config.hiddenSize
3115
+ );
3116
+ if (tracePrefillEnabled) {
3117
+ trace.perf('Prefill chunk submitted', {
3118
+ layer: l,
3119
+ elapsedMs: performance.now() - layerLoopTraceStart,
3120
+ prefillRecordMs,
3121
+ prefillSubmitWaitMs,
3122
+ });
3123
+ }
2071
3124
  currentRecorder = createRecorder('prefill-chunk');
2072
3125
  }
2073
3126
  }
3127
+ if (tracePrefillEnabled) {
3128
+ trace.perf('Prefill layer loop recorded', {
3129
+ numTokens,
3130
+ elapsedMs: performance.now() - layerLoopTraceStart,
3131
+ prefillRecordMs,
3132
+ prefillSubmitWaitMs,
3133
+ });
3134
+ }
2074
3135
  } finally {
2075
3136
  context.perLayerInputBuffer = null;
2076
3137
  if (perLayerInputs) {
@@ -2182,6 +3243,7 @@ export class PipelineGenerator {
2182
3243
  && !this.#state.disableRecordedLogits
2183
3244
  && numTokens === 1;
2184
3245
  if (currentRecorder && canRecordLogits) {
3246
+ const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
2185
3247
  const recorded = await recordLogitsGPU(
2186
3248
  currentRecorder,
2187
3249
  currentHiddenBuffer,
@@ -2202,6 +3264,13 @@ export class PipelineGenerator {
2202
3264
  const logitsData = await readBufferSlice(recorded.logitsBuffer, lastLogitsOffset, lastLogitsSize);
2203
3265
  releaseBuffer(recorded.logitsBuffer);
2204
3266
  lastLogits = decodeReadback(logitsData, recorded.logitsDtype);
3267
+ if (tracePrefillEnabled) {
3268
+ trace.perf('Prefill recorded logits complete', {
3269
+ numTokens,
3270
+ vocabSize: logitsVocabSize,
3271
+ elapsedMs: performance.now() - logitsTraceStart,
3272
+ });
3273
+ }
2205
3274
 
2206
3275
  const health = getLogitsHealth(lastLogits);
2207
3276
  if (health.nanCount > 0 || health.infCount > 0 || health.nonZeroCount === 0) {
@@ -2240,6 +3309,7 @@ export class PipelineGenerator {
2240
3309
 
2241
3310
  releaseBuffer(currentHiddenBuffer);
2242
3311
  } else {
3312
+ const logitsTraceStart = tracePrefillEnabled ? performance.now() : 0;
2243
3313
  if (currentRecorder) {
2244
3314
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2245
3315
  currentHiddenBuffer,
@@ -2267,6 +3337,13 @@ export class PipelineGenerator {
2267
3337
  ? logits
2268
3338
  : extractLastPositionLogits(logits, numTokens, logitsVocabSize);
2269
3339
  releaseBuffer(currentHiddenBuffer);
3340
+ if (tracePrefillEnabled) {
3341
+ trace.perf('Prefill logits complete', {
3342
+ numTokens,
3343
+ vocabSize: logitsVocabSize,
3344
+ elapsedMs: performance.now() - logitsTraceStart,
3345
+ });
3346
+ }
2270
3347
  }
2271
3348
 
2272
3349
  this.#state.currentSeqLen = startPos + numTokens;
@@ -2286,6 +3363,9 @@ export class PipelineGenerator {
2286
3363
  if (opts.debug) {
2287
3364
  logitsSanity(lastLogits, 'Prefill', (tokens) => resolveTokenText(this.#state.tokenizer, tokens));
2288
3365
  }
3366
+ if (isTraceEnabled('logits')) {
3367
+ trace.logits('PREFILL_LOGITS_HEALTH', getLogitsHealth(lastLogits));
3368
+ }
2289
3369
 
2290
3370
  if (opts.debug) {
2291
3371
  if (this.#state.kvCache?.hasGPUCache?.()) {
@@ -2295,6 +3375,13 @@ export class PipelineGenerator {
2295
3375
  }
2296
3376
  }
2297
3377
 
3378
+ if (tracePrefillEnabled) {
3379
+ trace.perf('Prefill phase complete', {
3380
+ numTokens,
3381
+ totalMs: performance.now() - prefillTraceStart,
3382
+ });
3383
+ }
3384
+
2298
3385
  return lastLogits;
2299
3386
  }
2300
3387