doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -14,15 +14,11 @@ import {
14
14
  validateCaptureConfig,
15
15
  } from '../../../debug/index.js';
16
16
  import { validateCallTimeOptions } from '../../../config/param-validator.js';
17
- import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
18
17
  import { selectRuleValue } from '../../../rules/rule-registry.js';
19
- import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
20
- import { f32ToF16Array } from '../../kv-cache/types.js';
21
18
 
22
19
  // Pipeline sub-modules
23
20
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
24
- import { applyChatTemplate, createKVCache, isStopToken } from './init.js';
25
- import { formatChatMessages } from './chat-format.js';
21
+ import { createKVCache, isStopToken } from './init.js';
26
22
  import { embed } from './embed.js';
27
23
  import { processLayer } from './layer.js';
28
24
  import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
@@ -43,6 +39,7 @@ import {
43
39
  debugCheckBuffer as debugCheckBufferHelper,
44
40
  getLogitsConfig,
45
41
  getLogitsWeights,
42
+ resolvePerLayerInputsSession,
46
43
  releaseSharedAttentionState,
47
44
  } from './generator-helpers.js';
48
45
  import {
@@ -61,12 +58,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
61
58
  import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
62
59
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
63
60
  import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
61
+ import { drainPendingTsirReads } from './tsir-fixture-writer.js';
64
62
  import {
65
63
  activateFallbackExecutionPlan,
66
64
  hasFallbackExecutionPlan,
67
65
  rebaseExecutionSessionPlan,
68
66
  resetActiveExecutionPlan,
69
67
  resolveMaxBatchDecodeTokens,
68
+ resolvePrefillRecorderChunkLayers,
70
69
  resolveActiveExecutionPlan,
71
70
  setActiveExecutionPlan,
72
71
  } from './execution-plan.js';
@@ -80,15 +79,37 @@ import {
80
79
  preparePerLayerInputs,
81
80
  createPleBufferCache,
82
81
  prefetchPerLayerRow,
83
- ensurePleScaledProjectionNormWeight,
84
- ensurePleGpuHotVocabularyRuntime,
85
- ensurePleGpuSplitTablesRuntime,
86
82
  getPleHotVocabularyRuntime,
87
83
  hasGpuSplitPerLayerInputEmbeddings,
88
84
  hasRangeBackedPerLayerInputEmbeddings,
89
85
  } from './per-layer-inputs.js';
90
86
  import { createTensor } from '../../../gpu/tensor.js';
91
- import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
87
+
88
+ // Extracted standalone helpers
89
+ import {
90
+ resolvePromptInput,
91
+ releasePerLayerInputBuffer,
92
+ normalizePrefixEmbeddingOverride,
93
+ resolvePrefillEmbeddingInputIds,
94
+ resolvePrefillMultimodalBidirectionalSpan,
95
+ applyPrefixEmbeddingOverride,
96
+ shouldDisablePrefillCommandBatching,
97
+ } from './generator-prefill-helpers.js';
98
+ import {
99
+ shouldDisableBatchDecodeAfterShortBatch,
100
+ resolveHotVocabularyBatchDecodeAvailability,
101
+ primePleDecodeRuntimeCache,
102
+ recordPrefillProfileStep,
103
+ resolveTokenText,
104
+ usesReplayPrefillDecode,
105
+ assertIncrementalDecodeSupport,
106
+ summarizeExecutionPlan,
107
+ shouldRetryWithFinitenessFallback,
108
+ createUnhandledFinitenessPolicyError,
109
+ resolveTargetPlanKVDtype,
110
+ resolveCurrentKVCacheDtype,
111
+ cloneRuntimeInferenceWithKVDtype,
112
+ } from './generator-decode-policy.js';
92
113
 
93
114
  let intentBundleModulePromise = null;
94
115
 
@@ -97,507 +118,6 @@ async function getExperimentalIntentBundleModule() {
97
118
  return intentBundleModulePromise;
98
119
  }
99
120
 
100
- function isStructuredChatRequest(prompt) {
101
- return prompt != null
102
- && typeof prompt === 'object'
103
- && !Array.isArray(prompt)
104
- && Array.isArray(prompt.messages);
105
- }
106
-
107
- export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
108
- return hitStop !== true
109
- && Number.isInteger(actualCount)
110
- && Number.isInteger(requestedCount)
111
- && actualCount > 0
112
- && actualCount < requestedCount;
113
- }
114
-
115
- export function resolveHotVocabularyBatchDecodeAvailability({
116
- hasRangeBackedPerLayerInputs,
117
- pleHotVocabularyRuntime,
118
- tokenId,
119
- }) {
120
- if (hasRangeBackedPerLayerInputs !== true) {
121
- return false;
122
- }
123
- if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
124
- return false;
125
- }
126
- const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
127
- if (!Number.isInteger(sentinelIndex)) {
128
- return false;
129
- }
130
- if (!Number.isInteger(tokenId)) {
131
- return false;
132
- }
133
- const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
134
- return hotTokenIndex !== sentinelIndex;
135
- }
136
-
137
- async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
138
- await ensurePleGpuHotVocabularyRuntime({
139
- config: state.modelConfig,
140
- weights: state.weights,
141
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
142
- debugFlags: state.debugFlags,
143
- tokenizer: state.tokenizer ?? null,
144
- seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
145
- });
146
- await ensurePleGpuSplitTablesRuntime({
147
- config: state.modelConfig,
148
- weights: state.weights,
149
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
150
- debugFlags: state.debugFlags,
151
- });
152
- await ensurePleScaledProjectionNormWeight({
153
- config: state.modelConfig,
154
- weights: state.weights,
155
- weightConfig: {
156
- rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
157
- },
158
- debugFlags: state.debugFlags,
159
- });
160
- }
161
-
162
- function recordPrefillProfileStep(state, entry) {
163
- if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
164
- if (!state.stats.prefillProfileSteps) {
165
- state.stats.prefillProfileSteps = [];
166
- }
167
- state.stats.prefillProfileSteps.push(entry);
168
- }
169
-
170
- function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
171
- const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
172
- if (typeof prompt === 'string') {
173
- if (useChatTemplate && state.modelConfig.chatTemplateType) {
174
- if (state.modelConfig.chatTemplateType === 'translategemma') {
175
- throw new Error(
176
- `[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
177
- 'Pass { messages: [...] } instead of a plain string prompt.'
178
- );
179
- }
180
- return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
181
- }
182
- return prompt;
183
- }
184
-
185
- if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
186
- throw new Error(
187
- `[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
188
- 'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
189
- );
190
- }
191
- const messages = isStructuredChatRequest(prompt)
192
- ? prompt.messages
193
- : (Array.isArray(prompt) ? prompt : null);
194
- if (!messages) {
195
- throw new Error(
196
- `[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
197
- );
198
- }
199
- const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
200
- return formatChatMessages(messages, templateType, chatOptions);
201
- }
202
-
203
- function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
204
- if (!buffer) {
205
- return;
206
- }
207
- const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
208
- if (ownsBuffer) {
209
- return;
210
- }
211
- const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
212
- if (cachedPleBuffer) {
213
- return;
214
- }
215
- if (recorder) {
216
- recorder.trackTemporaryBuffer(buffer);
217
- return;
218
- }
219
- releaseBuffer(buffer);
220
- }
221
-
222
- function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
223
- if (override == null) {
224
- return null;
225
- }
226
- if (typeof override !== 'object') {
227
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
228
- }
229
-
230
- const prefixLength = Number(override.prefixLength);
231
- if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
232
- throw new Error(
233
- `[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
234
- );
235
- }
236
- if (prefixLength === 0) {
237
- return null;
238
- }
239
-
240
- const offset = Number(override.offset ?? 0);
241
- if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
242
- throw new Error(
243
- `[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
244
- );
245
- }
246
- if (offset + prefixLength > numTokens) {
247
- throw new Error(
248
- `[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
249
- );
250
- }
251
-
252
- const embeddings = override.embeddings ?? null;
253
- if (!embeddings) {
254
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
255
- }
256
-
257
- const expectedLength = prefixLength * hiddenSize;
258
- if (embeddings instanceof Float32Array) {
259
- if (embeddings.length !== expectedLength) {
260
- throw new Error(
261
- `[Pipeline] ${contextLabel}: embedding override length mismatch ` +
262
- `(expected=${expectedLength}, got=${embeddings.length}).`
263
- );
264
- }
265
- } else if (isGpuBufferInstance(embeddings)) {
266
- const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
267
- if (embeddings.size < expectedBytes) {
268
- throw new Error(
269
- `[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
270
- `(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
271
- );
272
- }
273
- } else {
274
- throw new Error(
275
- `[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
276
- );
277
- }
278
-
279
- return {
280
- prefixLength,
281
- offset,
282
- embeddings,
283
- expectedLength,
284
- byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
285
- byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
286
- };
287
- }
288
-
289
- function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
290
- if (embeddingInputSpan == null) {
291
- return inputIds;
292
- }
293
- if (typeof embeddingInputSpan !== 'object') {
294
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
295
- }
296
-
297
- const offset = Number(embeddingInputSpan.offset);
298
- const length = Number(embeddingInputSpan.length);
299
- const tokenId = Number(embeddingInputSpan.tokenId);
300
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
301
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
302
- }
303
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
304
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
305
- }
306
- if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
307
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
308
- }
309
- if (offset + length > inputIds.length) {
310
- throw new Error(
311
- `[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
312
- `exceeds inputIds length ${inputIds.length}.`
313
- );
314
- }
315
- const replacedInputIds = Array.from(inputIds);
316
- replacedInputIds.fill(tokenId, offset, offset + length);
317
- return replacedInputIds;
318
- }
319
-
320
- function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
321
- if (bidirectionalSpan == null) {
322
- return null;
323
- }
324
- if (typeof bidirectionalSpan !== 'object') {
325
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
326
- }
327
-
328
- const offset = Number(bidirectionalSpan.offset);
329
- const length = Number(bidirectionalSpan.length);
330
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
331
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
332
- }
333
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
334
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
335
- }
336
- if ((offset + length) > inputIds.length) {
337
- throw new Error(
338
- `[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
339
- `exceeds inputIds length ${inputIds.length}.`
340
- );
341
- }
342
- return { offset, length };
343
- }
344
-
345
- async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
346
- if (!override) {
347
- return baseTensor;
348
- }
349
- if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
350
- throw new Error(
351
- `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
352
- );
353
- }
354
-
355
- const device = getDevice();
356
- if (!device) {
357
- throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
358
- }
359
-
360
- const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
361
- const targetElementOffset = override.offset * hiddenSize;
362
- const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
363
- try {
364
- const encoder = device.createCommandEncoder();
365
- encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
366
- if (baseTensor.dtype === 'f32') {
367
- if (isGpuBufferInstance(override.embeddings)) {
368
- encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
369
- device.queue.submit([encoder.finish()]);
370
- } else {
371
- device.queue.submit([encoder.finish()]);
372
- uploadData(outputBuffer, override.embeddings, override.byteOffset);
373
- }
374
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
375
- }
376
-
377
- if (isGpuBufferInstance(override.embeddings)) {
378
- const overrideTensor = createTensor(
379
- override.embeddings,
380
- 'f32',
381
- [override.prefixLength, hiddenSize],
382
- 'prefill_embedding_override_f32'
383
- );
384
- assertImplicitDtypeTransitionAllowed({
385
- executionPolicies,
386
- fromDtype: 'f32',
387
- toDtype: 'f16',
388
- op: 'prefill_embedding_override',
389
- detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
390
- });
391
- const castedOverride = await castF32ToF16(overrideTensor);
392
- try {
393
- encoder.copyBufferToBuffer(
394
- castedOverride.buffer,
395
- 0,
396
- outputBuffer,
397
- targetByteOffset,
398
- castedOverride.buffer.size
399
- );
400
- device.queue.submit([encoder.finish()]);
401
- } finally {
402
- releaseBuffer(castedOverride.buffer);
403
- }
404
- } else {
405
- assertImplicitDtypeTransitionAllowed({
406
- executionPolicies,
407
- fromDtype: 'f32',
408
- toDtype: 'f16',
409
- op: 'prefill_embedding_override',
410
- detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
411
- });
412
- const packedOverride = f32ToF16Array(override.embeddings);
413
- device.queue.submit([encoder.finish()]);
414
- uploadData(outputBuffer, packedOverride, targetByteOffset);
415
- }
416
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
417
- } catch (error) {
418
- releaseBuffer(outputBuffer);
419
- throw error;
420
- }
421
- }
422
-
423
- /**
424
- * Resolve display text for a token ID sequence.
425
- *
426
- * Preference order:
427
- * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
428
- * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
429
- * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
430
- * unless primary returned empty string and fallback looks like a special token (<...>),
431
- * in which case empty string is preserved to keep skip-special behavior deterministic.
432
- * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
433
- * 5. fallbackText — static fallback string (default '?').
434
- */
435
- function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
436
- const renderPrimary = typeof renderTokenText === 'function'
437
- ? renderTokenText
438
- : (ids) => tokenizer?.decode?.(ids);
439
- const renderFallback = typeof renderFallbackTokenText === 'function'
440
- ? renderFallbackTokenText
441
- : (ids) => tokenizer?.decode?.(ids, false);
442
-
443
- const primaryText = renderPrimary(tokenIds);
444
- if (typeof primaryText === 'string' && primaryText.length > 0) {
445
- return primaryText;
446
- }
447
-
448
- const fallback = renderFallback(tokenIds);
449
- if (typeof fallback === 'string' && fallback.length > 0) {
450
- // Keep skip-special behavior deterministic: if primary decoding filtered this
451
- // token to empty, do not reintroduce obvious special-token text via fallback.
452
- if (
453
- primaryText === ''
454
- && /^<[^>\n]{1,80}>$/.test(fallback.trim())
455
- ) {
456
- return '';
457
- }
458
- return fallback;
459
- }
460
-
461
- return fallbackText;
462
- }
463
-
464
- function usesReplayPrefillDecode(state) {
465
- return state?.modelConfig?.decodeStrategy === 'replay_prefill';
466
- }
467
-
468
- export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
469
- if (
470
- opts?.disableCommandBatching === true
471
- || opts?.debug === true
472
- || (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
473
- ) {
474
- return true;
475
- }
476
- if (state?.kvCache?.layout === 'bdpa_paged') {
477
- return true;
478
- }
479
- if (multimodalBidirectionalSpan == null) {
480
- return false;
481
- }
482
- if (state?.kvCache?.hasGPUCache?.() !== true) {
483
- return false;
484
- }
485
- // WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
486
- // multimodal logits on the first generated token. Keep this lane fail-closed
487
- // until the recorder/cache interaction is numerically matched to the direct path.
488
- return true;
489
- }
490
-
491
- function assertIncrementalDecodeSupport(state, operation) {
492
- if (!usesReplayPrefillDecode(state)) {
493
- return;
494
- }
495
- throw new Error(
496
- `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
497
- 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
498
- 'explicit layerTypes for mixed-geometry/shared-KV decode.'
499
- );
500
- }
501
-
502
- function summarizeExecutionPlan(plan) {
503
- if (!plan) {
504
- return null;
505
- }
506
- if (typeof plan !== 'object') {
507
- log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
508
- return null;
509
- }
510
- if (typeof plan.id !== 'string') {
511
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
512
- }
513
- if (typeof plan.activationDtype !== 'string') {
514
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
515
- }
516
- return {
517
- id: plan.id,
518
- kernelPathId: plan.kernelPathId ?? null,
519
- kernelPathSource: plan.kernelPathSource ?? 'none',
520
- activationDtype: plan.activationDtype,
521
- readbackInterval: plan.readbackInterval ?? null,
522
- batchSize: plan.defaultBatchSize,
523
- stopCheckMode: plan.defaultStopCheckMode,
524
- ringTokens: plan.ringTokens ?? null,
525
- ringStop: plan.ringStop ?? null,
526
- ringStaging: plan.ringStaging ?? null,
527
- };
528
- }
529
-
530
- export function shouldRetryWithFinitenessFallback(error) {
531
- if (error?.name === 'FinitenessError') {
532
- return true;
533
- }
534
- const message = typeof error?.message === 'string'
535
- ? error.message
536
- : (typeof error === 'string' ? error : '');
537
- if (!message.startsWith('[Sampling]')) {
538
- return false;
539
- }
540
- return message.includes('no finite candidate logits after masking the pad token')
541
- || message.includes('Softmax produced no finite candidate probabilities');
542
- }
543
-
544
- function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
545
- const activePlan = resolveActiveExecutionPlan(state);
546
- const wrapped = new Error(
547
- `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
548
- `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
549
- 'Resolve the unstable path with an explicit capability-aware execution override, ' +
550
- 'or opt into alternate-plan recovery with ' +
551
- 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
552
- error instanceof Error ? { cause: error } : undefined
553
- );
554
- wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
555
- return wrapped;
556
- }
557
-
558
- function assertResolvedKVDtype(kvDtype, contextLabel) {
559
- if (kvDtype === 'f16' || kvDtype === 'f32') {
560
- return kvDtype;
561
- }
562
- throw new Error(
563
- `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
564
- `got ${JSON.stringify(kvDtype)}.`
565
- );
566
- }
567
-
568
- function resolveTargetPlanKVDtype(plan, contextLabel) {
569
- return assertResolvedKVDtype(
570
- plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
571
- contextLabel
572
- );
573
- }
574
-
575
- function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
576
- return assertResolvedKVDtype(
577
- state?.kvCache?.kvDtype
578
- ?? plan?.kernelPath?.kvDtype
579
- ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
580
- ?? plan?.activationDtype
581
- ?? null,
582
- contextLabel
583
- );
584
- }
585
-
586
- function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
587
- const runtimeInference = state?.runtimeConfig?.inference;
588
- if (!runtimeInference?.session?.kvcache) {
589
- throw new Error(
590
- '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
591
- );
592
- }
593
- return mergeRuntimeValues(runtimeInference, {
594
- session: {
595
- kvcache: {
596
- kvDtype,
597
- },
598
- },
599
- });
600
- }
601
121
 
602
122
  export class PipelineGenerator {
603
123
 
@@ -797,14 +317,36 @@ export class PipelineGenerator {
797
317
  _sampleNextTokenFromLogits(logits, generatedIds, opts) {
798
318
  const sampledLogits = Float32Array.from(logits);
799
319
  applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
320
+ // Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
321
+ // implement grammar/schema-constrained decoding. The hook receives the
322
+ // mutable logit buffer (after repetition penalty) plus the running token
323
+ // sequence so it can track parse state across decode steps.
324
+ if (typeof opts?.logitMaskFn === "function") {
325
+ try {
326
+ opts.logitMaskFn(sampledLogits, {
327
+ generatedIds,
328
+ tokenizer: this.#state.tokenizer ?? null,
329
+ vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
330
+ });
331
+ } catch (maskError) {
332
+ log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
333
+ }
334
+ }
800
335
  const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
801
- return sample(sampledLogits, {
336
+ const tokenId = sample(sampledLogits, {
802
337
  temperature: opts.temperature,
803
338
  topP: opts.topP,
804
339
  topK: opts.topK,
805
340
  padTokenId,
806
341
  seed: opts.seed,
807
342
  });
343
+ if (typeof opts.onLogits === 'function') {
344
+ opts.onLogits(sampledLogits, {
345
+ tokenId,
346
+ inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
347
+ });
348
+ }
349
+ return tokenId;
808
350
  }
809
351
 
810
352
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
@@ -865,6 +407,11 @@ export class PipelineGenerator {
865
407
  return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
866
408
  }
867
409
 
410
+ _recordStopReason(reason, tokenId = null) {
411
+ this.#state.stats.stopReason = reason;
412
+ this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
413
+ }
414
+
868
415
  async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
869
416
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
870
417
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
@@ -878,6 +425,8 @@ export class PipelineGenerator {
878
425
  this.#state.stats.decodeRecordMs = 0;
879
426
  this.#state.stats.decodeSubmitWaitMs = 0;
880
427
  this.#state.stats.decodeReadbackWaitMs = 0;
428
+ this.#state.stats.prefillRecordMs = 0;
429
+ this.#state.stats.prefillSubmitWaitMs = 0;
881
430
  this.#state.stats.singleTokenSubmitWaitMs = 0;
882
431
  this.#state.stats.singleTokenReadbackWaitMs = 0;
883
432
  this.#state.stats.singleTokenOrchestrationMs = 0;
@@ -889,10 +438,14 @@ export class PipelineGenerator {
889
438
  this.#state.stats.plePreparedTokenCacheBytes = 0;
890
439
  this.#state.stats.decodeMode = null;
891
440
  this.#state.stats.batchGuardReason = null;
441
+ this.#state.stats.stopReason = null;
442
+ this.#state.stats.stopTokenId = null;
892
443
  this.#state.stats.ttftMs = 0;
893
444
  const startTime = performance.now();
894
445
 
895
446
  const opts = resolveGenerateOptions(this.#state, options);
447
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
448
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
896
449
  // Validate and normalize sampling parameters through single source of truth
897
450
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
898
451
  opts.temperature = samplingConfig.temperature;
@@ -901,7 +454,8 @@ export class PipelineGenerator {
901
454
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
902
455
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
903
456
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
904
- if (diagnosticsEnabled) {
457
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
458
+ if (diagnosticsEnabled || tsirFixtureCfg) {
905
459
  const captureConfig = {
906
460
  ...createDefaultCaptureConfig(),
907
461
  enabled: true,
@@ -910,13 +464,20 @@ export class PipelineGenerator {
910
464
  };
911
465
  validateCaptureConfig(captureConfig);
912
466
  this.#state.operatorDiagnostics = {
913
- enabled: true,
467
+ enabled: diagnosticsEnabled === true,
914
468
  captureConfig,
915
- emitter: new OperatorEventEmitter({
469
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
916
470
  modelHash: this.#state.manifest?.modelId ?? null,
917
471
  runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
918
472
  executionPlanHash: opts.executionPlan?.id ?? null,
919
- }),
473
+ }) : null,
474
+ tsirFixture: tsirFixtureCfg ? {
475
+ dir: tsirFixtureCfg.dir,
476
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
477
+ ? tsirFixtureCfg.layerFilter
478
+ : null,
479
+ records: [],
480
+ } : null,
920
481
  };
921
482
  }
922
483
  const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
@@ -1036,6 +597,7 @@ export class PipelineGenerator {
1036
597
  yield* emitToken(this, firstToken, decodeToken);
1037
598
 
1038
599
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
600
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1039
601
  this.#state.stats.decodeTimeMs = 0;
1040
602
  this.#state.stats.tokensGenerated = 1;
1041
603
  this.#state.stats.decodeTokens = 1;
@@ -1262,6 +824,31 @@ export class PipelineGenerator {
1262
824
  // Generation Public API
1263
825
  // ==========================================================================
1264
826
 
827
+ /**
828
+ * Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
829
+ * match. Intended for "prefix-reuse" workflows where a caller wants to run
830
+ * several decodes that share a common prompt prefix: prefill once with the
831
+ * shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
832
+ * drop the tail's KV entries and reuse the prefix KV for the next tail.
833
+ *
834
+ * Only valid when no decode is in progress.
835
+ */
836
+ resetToSeqLen(seqLen) {
837
+ if (this.#state.isGenerating) {
838
+ throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
839
+ }
840
+ const target = Math.max(0, Math.floor(Number(seqLen) || 0));
841
+ if (!Number.isFinite(target)) {
842
+ throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
843
+ }
844
+ if (target > this.#state.currentSeqLen) {
845
+ throw new Error(
846
+ `InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
847
+ );
848
+ }
849
+ this.#state.kvCache?.truncate?.(target);
850
+ this.#state.currentSeqLen = target;
851
+ }
1265
852
 
1266
853
  async *generate(prompt, options = {}) {
1267
854
  yield* this._generateTokensInternal(prompt, options, 'text');
@@ -1284,6 +871,8 @@ export class PipelineGenerator {
1284
871
  this.#state.stats.decodeRecordMs = 0;
1285
872
  this.#state.stats.decodeSubmitWaitMs = 0;
1286
873
  this.#state.stats.decodeReadbackWaitMs = 0;
874
+ this.#state.stats.prefillRecordMs = 0;
875
+ this.#state.stats.prefillSubmitWaitMs = 0;
1287
876
  this.#state.stats.singleTokenSubmitWaitMs = 0;
1288
877
  this.#state.stats.singleTokenReadbackWaitMs = 0;
1289
878
  this.#state.stats.singleTokenOrchestrationMs = 0;
@@ -1295,15 +884,46 @@ export class PipelineGenerator {
1295
884
  this.#state.stats.plePreparedTokenCacheBytes = 0;
1296
885
  this.#state.stats.decodeMode = null;
1297
886
  this.#state.stats.batchGuardReason = null;
887
+ this.#state.stats.stopReason = null;
888
+ this.#state.stats.stopTokenId = null;
1298
889
  this.#state.stats.ttftMs = 0;
1299
890
  const startTime = performance.now();
1300
891
  const opts = resolveGenerateOptions(this.#state, options);
892
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
1301
893
  // Validate and normalize sampling parameters through single source of truth
1302
894
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
1303
895
  opts.temperature = samplingConfig.temperature;
1304
896
  opts.topP = samplingConfig.topP;
1305
897
  opts.topK = samplingConfig.topK;
1306
898
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
899
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
900
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
901
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
902
+ if (diagnosticsEnabled || tsirFixtureCfg) {
903
+ const captureConfig = {
904
+ ...createDefaultCaptureConfig(),
905
+ enabled: true,
906
+ defaultLevel: CAPTURE_LEVELS.SLICE,
907
+ ...(options?.diagnostics?.captureConfig ?? {}),
908
+ };
909
+ validateCaptureConfig(captureConfig);
910
+ this.#state.operatorDiagnostics = {
911
+ enabled: diagnosticsEnabled === true,
912
+ captureConfig,
913
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
914
+ modelHash: this.#state.manifest?.modelId ?? null,
915
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
916
+ executionPlanHash: opts.executionPlan?.id ?? null,
917
+ }) : null,
918
+ tsirFixture: tsirFixtureCfg ? {
919
+ dir: tsirFixtureCfg.dir,
920
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
921
+ ? tsirFixtureCfg.layerFilter
922
+ : null,
923
+ records: [],
924
+ } : null,
925
+ };
926
+ }
1307
927
 
1308
928
  try {
1309
929
  const prefillStartSeqLen = this.#state.currentSeqLen;
@@ -1353,6 +973,7 @@ export class PipelineGenerator {
1353
973
  tokenIds.push(tokenId);
1354
974
  }
1355
975
  } else {
976
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1356
977
  this.#state.stats.decodeTimeMs = 0;
1357
978
  this.#state.stats.tokensGenerated = 1;
1358
979
  this.#state.stats.decodeTokens = 1;
@@ -1370,6 +991,14 @@ export class PipelineGenerator {
1370
991
  this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
1371
992
  }
1372
993
  resetActiveExecutionPlan(this.#state);
994
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
995
+ ? {
996
+ enabled: true,
997
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
998
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
999
+ }
1000
+ : null;
1001
+ this.#state.operatorDiagnostics = null;
1373
1002
  this.#state.isGenerating = false;
1374
1003
  }
1375
1004
  }
@@ -1683,6 +1312,7 @@ export class PipelineGenerator {
1683
1312
  if (options.onToken) options.onToken(firstToken, firstText);
1684
1313
 
1685
1314
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1315
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1686
1316
  this.#state.stats.decodeTimeMs = 0;
1687
1317
  this.#state.stats.tokensGenerated = 1;
1688
1318
  this.#state.stats.decodeTokens = 1;
@@ -1729,6 +1359,10 @@ export class PipelineGenerator {
1729
1359
  }
1730
1360
 
1731
1361
  const decodeStart = performance.now();
1362
+ const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
1363
+ this.#state.modelConfig.perLayerInputsSession ?? null,
1364
+ this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
1365
+ );
1732
1366
  const lmHead = this.#state.weights.get('lm_head');
1733
1367
  const embedBuffer = this.#state.weights.get('embed');
1734
1368
  const hasCpuWeights = isCpuWeightBuffer(lmHead)
@@ -1805,7 +1439,10 @@ export class PipelineGenerator {
1805
1439
  }
1806
1440
 
1807
1441
  while (tokensGenerated < opts.maxTokens) {
1808
- if (options.signal?.aborted) break;
1442
+ if (options.signal?.aborted) {
1443
+ this._recordStopReason('aborted');
1444
+ break;
1445
+ }
1809
1446
  if (this._hasFinitenessFallbackWindow() && useBatchPath) {
1810
1447
  useBatchPath = false;
1811
1448
  }
@@ -1854,9 +1491,15 @@ export class PipelineGenerator {
1854
1491
  if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
1855
1492
  if (opts.stopSequences.length > 0) {
1856
1493
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1857
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1494
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1495
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
1496
+ break;
1497
+ }
1498
+ }
1499
+ if (hitStop) {
1500
+ this._recordStopReason('stop-token');
1501
+ break;
1858
1502
  }
1859
- if (hitStop) break;
1860
1503
  if (shouldDisableBatchDecodeAfterShortBatch({
1861
1504
  hitStop,
1862
1505
  actualCount: batchResult.actualCount,
@@ -1894,7 +1537,10 @@ export class PipelineGenerator {
1894
1537
  if (options.onToken) options.onToken(nextToken, tokenText);
1895
1538
  }
1896
1539
  this._consumeFinitenessFallbackToken(opts);
1897
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
1540
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
1541
+ this._recordStopReason('stop-token', nextToken);
1542
+ break;
1543
+ }
1898
1544
  }
1899
1545
  } else if (opts.speculation?.mode === 'self') {
1900
1546
  // Self-speculation: decode one base token plus a configurable burst of
@@ -1932,11 +1578,17 @@ export class PipelineGenerator {
1932
1578
  }
1933
1579
  this._consumeFinitenessFallbackToken(opts);
1934
1580
 
1935
- if (isStopToken(baseToken, stopTokenIds, eosToken)) break;
1581
+ if (isStopToken(baseToken, stopTokenIds, eosToken)) {
1582
+ this._recordStopReason('stop-token', baseToken);
1583
+ break;
1584
+ }
1936
1585
  if (tokensGenerated >= opts.maxTokens) break;
1937
1586
  if (opts.stopSequences.length > 0) {
1938
1587
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1939
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1588
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1589
+ this._recordStopReason('stop-sequence', baseToken);
1590
+ break;
1591
+ }
1940
1592
  }
1941
1593
 
1942
1594
  for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
@@ -1975,19 +1627,27 @@ export class PipelineGenerator {
1975
1627
  }
1976
1628
 
1977
1629
  if (isStopToken(specToken, stopTokenIds, eosToken)) {
1630
+ this._recordStopReason('stop-token', specToken);
1978
1631
  break;
1979
1632
  }
1980
1633
  if (opts.stopSequences.length > 0) {
1981
1634
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1982
1635
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1636
+ this._recordStopReason('stop-sequence', specToken);
1983
1637
  break;
1984
1638
  }
1985
1639
  }
1986
1640
  }
1987
- if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) break;
1641
+ if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
1642
+ this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
1643
+ break;
1644
+ }
1988
1645
  if (opts.stopSequences.length > 0) {
1989
1646
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1990
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1647
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1648
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
1649
+ break;
1650
+ }
1991
1651
  }
1992
1652
  } else {
1993
1653
  const tokenStart = performance.now();
@@ -2020,7 +1680,7 @@ export class PipelineGenerator {
2020
1680
  nextToken,
2021
1681
  pleWeights.embedTokensPerLayer,
2022
1682
  this.#state.modelConfig.numLayers * pleHiddenSize,
2023
- this.#state.runtimeConfig.inference.session?.perLayerInputs ?? this.#state.modelConfig.perLayerInputsSession,
1683
+ resolvedPerLayerInputsSession,
2024
1684
  );
2025
1685
  }
2026
1686
  }
@@ -2041,14 +1701,23 @@ export class PipelineGenerator {
2041
1701
  log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
2042
1702
  }
2043
1703
 
2044
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
1704
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
1705
+ this._recordStopReason('stop-token', nextToken);
1706
+ break;
1707
+ }
2045
1708
  if (opts.stopSequences.length > 0) {
2046
1709
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2047
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1710
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1711
+ this._recordStopReason('stop-sequence', nextToken);
1712
+ break;
1713
+ }
2048
1714
  }
2049
1715
  }
2050
1716
  }
2051
1717
 
1718
+ if (!this.#state.stats.stopReason) {
1719
+ this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
1720
+ }
2052
1721
  this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
2053
1722
  this.#state.stats.tokensGenerated = tokensGenerated;
2054
1723
  this.#state.stats.decodeTokens = tokensGenerated;
@@ -2214,6 +1883,7 @@ export class PipelineGenerator {
2214
1883
  context,
2215
1884
  {
2216
1885
  numTokens,
1886
+ pleCache: this.#state.pleCache ?? null,
2217
1887
  }
2218
1888
  );
2219
1889
  } catch (error) {
@@ -2273,16 +1943,35 @@ export class PipelineGenerator {
2273
1943
  // Chunked recorder submission: submit every N layers to release tracked intermediate
2274
1944
  // buffers, preventing unbounded memory growth during large prefills. Critical for
2275
1945
  // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2276
- const PREFILL_RECORDER_CHUNK_LAYERS = 4;
1946
+ const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
1947
+ hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
1948
+ numTokens,
1949
+ });
2277
1950
 
2278
1951
  let currentHiddenBuffer = hiddenStates.buffer;
1952
+ let prefillRecordMs = 0;
1953
+ let prefillSubmitWaitMs = 0;
2279
1954
  try {
2280
1955
  for (let l = 0; l < config.numLayers; l++) {
1956
+ // Per-layer hard cancellation: when the caller's AbortSignal aborts,
1957
+ // exit the prefill loop between layer dispatches rather than
1958
+ // continuing to burn GPU on superseded work. Granularity is one
1959
+ // layer (~5-50ms), which is the fastest cancel granularity WebGPU
1960
+ // exposes today.
1961
+ if (opts?.signal?.aborted) {
1962
+ const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
1963
+ const err = new Error(reason);
1964
+ err.name = "AbortError";
1965
+ err.code = "ABORT_ERR";
1966
+ throw err;
1967
+ }
2281
1968
  context.recorder = currentRecorder;
2282
1969
  context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
2283
1970
 
2284
1971
  const prevBuffer = currentHiddenBuffer;
1972
+ const layerRecordStart = performance.now();
2285
1973
  const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
1974
+ prefillRecordMs += performance.now() - layerRecordStart;
2286
1975
  if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
2287
1976
  currentHiddenBuffer = layerOutput;
2288
1977
  releasePerLayerInputBuffer(
@@ -2300,7 +1989,7 @@ export class PipelineGenerator {
2300
1989
  const isChunkBoundary = !isCheckpoint
2301
1990
  && currentRecorder
2302
1991
  && l < config.numLayers - 1
2303
- && (l + 1) % PREFILL_RECORDER_CHUNK_LAYERS === 0;
1992
+ && (l + 1) % prefillRecorderChunkLayers === 0;
2304
1993
 
2305
1994
  if (isCheckpoint && currentRecorder) {
2306
1995
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
@@ -2360,8 +2049,25 @@ export class PipelineGenerator {
2360
2049
  currentRecorder,
2361
2050
  'prefill_chunk_carry'
2362
2051
  );
2363
- await currentRecorder.submitAndWait();
2364
- await recordProfile(currentRecorder);
2052
+ // Chunk boundary exists only to bound intermediate buffer lifetime.
2053
+ // When the runtime opts into async chunk submit AND profile timings
2054
+ // are not being collected, skip the CPU-GPU wait: queue order is
2055
+ // preserved across submits and deferred cleanup still releases
2056
+ // tracked buffers when GPU work completes. Profile runs keep the
2057
+ // sync path because resolveProfileTimings requires the submitted
2058
+ // work to be done.
2059
+ const manifestSubmit = this.#state.modelConfig?.sessionSettings?.prefillChunkSubmitMode;
2060
+ const chunkSubmitMode = (manifestSubmit !== null && manifestSubmit !== undefined)
2061
+ ? manifestSubmit
2062
+ : (this.#state.runtimeConfig?.inference?.session?.prefillChunkSubmitMode ?? 'sync');
2063
+ const chunkSubmitStart = performance.now();
2064
+ if (chunkSubmitMode === 'async' && !opts.profile) {
2065
+ currentRecorder.submit({ cleanup: 'deferred' });
2066
+ } else {
2067
+ await currentRecorder.submitAndWait();
2068
+ await recordProfile(currentRecorder);
2069
+ }
2070
+ prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
2365
2071
  currentRecorder = createRecorder('prefill-chunk');
2366
2072
  }
2367
2073
  }
@@ -2378,8 +2084,26 @@ export class PipelineGenerator {
2378
2084
  }
2379
2085
  }
2380
2086
  releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
2087
+ this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
2088
+ this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
2381
2089
  }
2382
2090
 
2091
+ {
2092
+ const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
2093
+ if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
2094
+ if (currentRecorder) {
2095
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2096
+ currentHiddenBuffer,
2097
+ currentRecorder,
2098
+ 'prefill_tsir_drain_carry'
2099
+ );
2100
+ await currentRecorder.submitAndWait();
2101
+ await recordProfile(currentRecorder);
2102
+ currentRecorder = undefined;
2103
+ }
2104
+ await drainPendingTsirReads(tsirFixtureCfg);
2105
+ }
2106
+ }
2383
2107
  if (this.#state.finitenessBuffer) {
2384
2108
  if (currentRecorder) {
2385
2109
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(