doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -14,15 +14,11 @@ import {
14
14
  validateCaptureConfig,
15
15
  } from '../../../debug/index.js';
16
16
  import { validateCallTimeOptions } from '../../../config/param-validator.js';
17
- import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
18
17
  import { selectRuleValue } from '../../../rules/rule-registry.js';
19
- import { castF32ToF16 } from '../../../gpu/kernels/cast.js';
20
- import { f32ToF16Array } from '../../kv-cache/types.js';
21
18
 
22
19
  // Pipeline sub-modules
23
20
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
24
- import { applyChatTemplate, createKVCache, isStopToken } from './init.js';
25
- import { formatChatMessages } from './chat-format.js';
21
+ import { createKVCache, isStopToken } from './init.js';
26
22
  import { embed } from './embed.js';
27
23
  import { processLayer } from './layer.js';
28
24
  import { computeLogits, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from './logits/index.js';
@@ -62,12 +58,14 @@ import { resolveSamplingConfig } from './sampling-config.js';
62
58
  import { decodeReadback, getLogitsHealth } from './debug-utils/index.js';
63
59
  import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
64
60
  import { resolveDeferredRoundingWindowTokens } from './finiteness-policy.js';
61
+ import { drainPendingTsirReads } from './tsir-fixture-writer.js';
65
62
  import {
66
63
  activateFallbackExecutionPlan,
67
64
  hasFallbackExecutionPlan,
68
65
  rebaseExecutionSessionPlan,
69
66
  resetActiveExecutionPlan,
70
67
  resolveMaxBatchDecodeTokens,
68
+ resolvePrefillRecorderChunkLayers,
71
69
  resolveActiveExecutionPlan,
72
70
  setActiveExecutionPlan,
73
71
  } from './execution-plan.js';
@@ -81,15 +79,37 @@ import {
81
79
  preparePerLayerInputs,
82
80
  createPleBufferCache,
83
81
  prefetchPerLayerRow,
84
- ensurePleScaledProjectionNormWeight,
85
- ensurePleGpuHotVocabularyRuntime,
86
- ensurePleGpuSplitTablesRuntime,
87
82
  getPleHotVocabularyRuntime,
88
83
  hasGpuSplitPerLayerInputEmbeddings,
89
84
  hasRangeBackedPerLayerInputEmbeddings,
90
85
  } from './per-layer-inputs.js';
91
86
  import { createTensor } from '../../../gpu/tensor.js';
92
- import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
87
+
88
+ // Extracted standalone helpers
89
+ import {
90
+ resolvePromptInput,
91
+ releasePerLayerInputBuffer,
92
+ normalizePrefixEmbeddingOverride,
93
+ resolvePrefillEmbeddingInputIds,
94
+ resolvePrefillMultimodalBidirectionalSpan,
95
+ applyPrefixEmbeddingOverride,
96
+ shouldDisablePrefillCommandBatching,
97
+ } from './generator-prefill-helpers.js';
98
+ import {
99
+ shouldDisableBatchDecodeAfterShortBatch,
100
+ resolveHotVocabularyBatchDecodeAvailability,
101
+ primePleDecodeRuntimeCache,
102
+ recordPrefillProfileStep,
103
+ resolveTokenText,
104
+ usesReplayPrefillDecode,
105
+ assertIncrementalDecodeSupport,
106
+ summarizeExecutionPlan,
107
+ shouldRetryWithFinitenessFallback,
108
+ createUnhandledFinitenessPolicyError,
109
+ resolveTargetPlanKVDtype,
110
+ resolveCurrentKVCacheDtype,
111
+ cloneRuntimeInferenceWithKVDtype,
112
+ } from './generator-decode-policy.js';
93
113
 
94
114
  let intentBundleModulePromise = null;
95
115
 
@@ -98,521 +118,6 @@ async function getExperimentalIntentBundleModule() {
98
118
  return intentBundleModulePromise;
99
119
  }
100
120
 
101
- function isStructuredChatRequest(prompt) {
102
- return prompt != null
103
- && typeof prompt === 'object'
104
- && !Array.isArray(prompt)
105
- && Array.isArray(prompt.messages);
106
- }
107
-
108
- export function shouldDisableBatchDecodeAfterShortBatch({ hitStop, actualCount, requestedCount }) {
109
- return hitStop !== true
110
- && Number.isInteger(actualCount)
111
- && Number.isInteger(requestedCount)
112
- && actualCount > 0
113
- && actualCount < requestedCount;
114
- }
115
-
116
- export function resolveHotVocabularyBatchDecodeAvailability({
117
- hasRangeBackedPerLayerInputs,
118
- pleHotVocabularyRuntime,
119
- tokenId,
120
- }) {
121
- if (hasRangeBackedPerLayerInputs !== true) {
122
- return false;
123
- }
124
- if (!pleHotVocabularyRuntime || typeof pleHotVocabularyRuntime !== 'object') {
125
- return false;
126
- }
127
- const sentinelIndex = pleHotVocabularyRuntime.sentinelIndex;
128
- if (!Number.isInteger(sentinelIndex)) {
129
- return false;
130
- }
131
- if (!Number.isInteger(tokenId)) {
132
- return false;
133
- }
134
- const hotTokenIndex = pleHotVocabularyRuntime.hotTokenIndexMap?.[tokenId] ?? sentinelIndex;
135
- return hotTokenIndex !== sentinelIndex;
136
- }
137
-
138
- async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
139
- const perLayerInputsSession = resolvePerLayerInputsSession(
140
- state.modelConfig?.perLayerInputsSession ?? null,
141
- state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
142
- );
143
- const materialization = perLayerInputsSession?.materialization;
144
- if (state.debug) {
145
- log.debug(
146
- 'Pipeline',
147
- `PLE materialization session=${materialization ?? 'null'} ` +
148
- `modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
149
- `runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
150
- `manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
151
- );
152
- }
153
- await ensurePleGpuHotVocabularyRuntime({
154
- config: state.modelConfig,
155
- weights: state.weights,
156
- perLayerInputsSession,
157
- debugFlags: state.debugFlags,
158
- tokenizer: state.tokenizer ?? null,
159
- seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
160
- });
161
- await ensurePleGpuSplitTablesRuntime({
162
- config: state.modelConfig,
163
- weights: state.weights,
164
- perLayerInputsSession,
165
- debugFlags: state.debugFlags,
166
- });
167
- await ensurePleScaledProjectionNormWeight({
168
- config: state.modelConfig,
169
- weights: state.weights,
170
- weightConfig: {
171
- rmsNormWeightOffset: state.modelConfig?.rmsNormWeightOffset ?? false,
172
- },
173
- debugFlags: state.debugFlags,
174
- });
175
- }
176
-
177
- function recordPrefillProfileStep(state, entry) {
178
- if (!entry?.timings || Object.keys(entry.timings).length === 0) return;
179
- if (!state.stats.prefillProfileSteps) {
180
- state.stats.prefillProfileSteps = [];
181
- }
182
- state.stats.prefillProfileSteps.push(entry);
183
- }
184
-
185
- function resolvePromptInput(state, prompt, useChatTemplate, contextLabel) {
186
- const chatOptions = state.modelConfig.chatTemplateThinking === true ? { thinking: true } : undefined;
187
- if (typeof prompt === 'string') {
188
- if (useChatTemplate && state.modelConfig.chatTemplateType) {
189
- if (state.modelConfig.chatTemplateType === 'translategemma') {
190
- throw new Error(
191
- `[Pipeline] ${contextLabel}: translategemma chat template requires structured messages. ` +
192
- 'Pass { messages: [...] } instead of a plain string prompt.'
193
- );
194
- }
195
- return applyChatTemplate(prompt, state.modelConfig.chatTemplateType, chatOptions);
196
- }
197
- return prompt;
198
- }
199
-
200
- if (prompt != null && typeof prompt === 'object' && !Array.isArray(prompt) && 'messages' in prompt && !Array.isArray(prompt.messages)) {
201
- throw new Error(
202
- `[Pipeline] ${contextLabel}: prompt.messages must be an array of chat messages, got ${typeof prompt.messages}. ` +
203
- 'Pass { messages: [{ role: "user", content: "..." }, ...] }.'
204
- );
205
- }
206
- const messages = isStructuredChatRequest(prompt)
207
- ? prompt.messages
208
- : (Array.isArray(prompt) ? prompt : null);
209
- if (!messages) {
210
- throw new Error(
211
- `[Pipeline] ${contextLabel}: prompt must be a string, chat message array, or { messages: [...] }.`
212
- );
213
- }
214
- const templateType = useChatTemplate ? state.modelConfig.chatTemplateType : null;
215
- return formatChatMessages(messages, templateType, chatOptions);
216
- }
217
-
218
- function releasePerLayerInputBuffer(buffer, recorder, decodeBuffers, pleCache = null) {
219
- if (!buffer) {
220
- return;
221
- }
222
- const ownsBuffer = decodeBuffers?.ownsBuffer(buffer) ?? false;
223
- if (ownsBuffer) {
224
- return;
225
- }
226
- const cachedPleBuffer = pleCache?.ownedBuffers instanceof Set && pleCache.ownedBuffers.has(buffer);
227
- if (cachedPleBuffer) {
228
- return;
229
- }
230
- if (recorder) {
231
- recorder.trackTemporaryBuffer(buffer);
232
- return;
233
- }
234
- releaseBuffer(buffer);
235
- }
236
-
237
- function normalizePrefixEmbeddingOverride(override, hiddenSize, numTokens, contextLabel) {
238
- if (override == null) {
239
- return null;
240
- }
241
- if (typeof override !== 'object') {
242
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides must be an object when provided.`);
243
- }
244
-
245
- const prefixLength = Number(override.prefixLength);
246
- if (!Number.isFinite(prefixLength) || prefixLength < 0 || Math.floor(prefixLength) !== prefixLength) {
247
- throw new Error(
248
- `[Pipeline] ${contextLabel}: embeddingOverrides.prefixLength must be a non-negative integer.`
249
- );
250
- }
251
- if (prefixLength === 0) {
252
- return null;
253
- }
254
-
255
- const offset = Number(override.offset ?? 0);
256
- if (!Number.isFinite(offset) || offset < 0 || Math.floor(offset) !== offset) {
257
- throw new Error(
258
- `[Pipeline] ${contextLabel}: embeddingOverrides.offset must be a non-negative integer.`
259
- );
260
- }
261
- if (offset + prefixLength > numTokens) {
262
- throw new Error(
263
- `[Pipeline] ${contextLabel}: embedding override offset=${offset} + prefixLength=${prefixLength} exceeds numTokens=${numTokens}.`
264
- );
265
- }
266
-
267
- const embeddings = override.embeddings ?? null;
268
- if (!embeddings) {
269
- throw new Error(`[Pipeline] ${contextLabel}: embeddingOverrides.embeddings is required when prefixLength > 0.`);
270
- }
271
-
272
- const expectedLength = prefixLength * hiddenSize;
273
- if (embeddings instanceof Float32Array) {
274
- if (embeddings.length !== expectedLength) {
275
- throw new Error(
276
- `[Pipeline] ${contextLabel}: embedding override length mismatch ` +
277
- `(expected=${expectedLength}, got=${embeddings.length}).`
278
- );
279
- }
280
- } else if (isGpuBufferInstance(embeddings)) {
281
- const expectedBytes = expectedLength * Float32Array.BYTES_PER_ELEMENT;
282
- if (embeddings.size < expectedBytes) {
283
- throw new Error(
284
- `[Pipeline] ${contextLabel}: embedding override GPUBuffer too small ` +
285
- `(expected>=${expectedBytes} bytes, got=${embeddings.size}).`
286
- );
287
- }
288
- } else {
289
- throw new Error(
290
- `[Pipeline] ${contextLabel}: embeddingOverrides.embeddings must be a Float32Array or GPUBuffer.`
291
- );
292
- }
293
-
294
- return {
295
- prefixLength,
296
- offset,
297
- embeddings,
298
- expectedLength,
299
- byteLength: expectedLength * Float32Array.BYTES_PER_ELEMENT,
300
- byteOffset: offset * hiddenSize * Float32Array.BYTES_PER_ELEMENT,
301
- };
302
- }
303
-
304
- function resolvePrefillEmbeddingInputIds(inputIds, embeddingInputSpan, contextLabel) {
305
- if (embeddingInputSpan == null) {
306
- return inputIds;
307
- }
308
- if (typeof embeddingInputSpan !== 'object') {
309
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan must be an object when provided.`);
310
- }
311
-
312
- const offset = Number(embeddingInputSpan.offset);
313
- const length = Number(embeddingInputSpan.length);
314
- const tokenId = Number(embeddingInputSpan.tokenId);
315
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
316
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.offset must be a non-negative integer.`);
317
- }
318
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 0) {
319
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.length must be a non-negative integer.`);
320
- }
321
- if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
322
- throw new Error(`[Pipeline] ${contextLabel}: embeddingInputSpan.tokenId must be a non-negative integer.`);
323
- }
324
- if (offset + length > inputIds.length) {
325
- throw new Error(
326
- `[Pipeline] ${contextLabel}: embeddingInputSpan offset=${offset} + length=${length} ` +
327
- `exceeds inputIds length ${inputIds.length}.`
328
- );
329
- }
330
- const replacedInputIds = Array.from(inputIds);
331
- replacedInputIds.fill(tokenId, offset, offset + length);
332
- return replacedInputIds;
333
- }
334
-
335
- function resolvePrefillMultimodalBidirectionalSpan(inputIds, bidirectionalSpan, contextLabel) {
336
- if (bidirectionalSpan == null) {
337
- return null;
338
- }
339
- if (typeof bidirectionalSpan !== 'object') {
340
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan must be an object when provided.`);
341
- }
342
-
343
- const offset = Number(bidirectionalSpan.offset);
344
- const length = Number(bidirectionalSpan.length);
345
- if (!Number.isFinite(offset) || Math.floor(offset) !== offset || offset < 0) {
346
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.offset must be a non-negative integer.`);
347
- }
348
- if (!Number.isFinite(length) || Math.floor(length) !== length || length < 1) {
349
- throw new Error(`[Pipeline] ${contextLabel}: multimodalBidirectionalSpan.length must be a positive integer.`);
350
- }
351
- if ((offset + length) > inputIds.length) {
352
- throw new Error(
353
- `[Pipeline] ${contextLabel}: multimodalBidirectionalSpan offset=${offset} + length=${length} ` +
354
- `exceeds inputIds length ${inputIds.length}.`
355
- );
356
- }
357
- return { offset, length };
358
- }
359
-
360
- async function applyPrefixEmbeddingOverride(baseTensor, override, hiddenSize, contextLabel, executionPolicies = null) {
361
- if (!override) {
362
- return baseTensor;
363
- }
364
- if (baseTensor.dtype !== 'f32' && baseTensor.dtype !== 'f16') {
365
- throw new Error(
366
- `[Pipeline] ${contextLabel}: embedding overrides require f32 or f16 activations, got ${baseTensor.dtype}.`
367
- );
368
- }
369
-
370
- const device = getDevice();
371
- if (!device) {
372
- throw new Error(`[Pipeline] ${contextLabel}: GPU device is required for embedding overrides.`);
373
- }
374
-
375
- const outputBuffer = acquireBuffer(baseTensor.buffer.size, undefined, 'prefill_embedding_override');
376
- const targetElementOffset = override.offset * hiddenSize;
377
- const targetByteOffset = targetElementOffset * (baseTensor.dtype === 'f16' ? 2 : 4);
378
- try {
379
- const encoder = device.createCommandEncoder();
380
- encoder.copyBufferToBuffer(baseTensor.buffer, 0, outputBuffer, 0, baseTensor.buffer.size);
381
- if (baseTensor.dtype === 'f32') {
382
- if (isGpuBufferInstance(override.embeddings)) {
383
- encoder.copyBufferToBuffer(override.embeddings, 0, outputBuffer, override.byteOffset, override.byteLength);
384
- device.queue.submit([encoder.finish()]);
385
- } else {
386
- device.queue.submit([encoder.finish()]);
387
- uploadData(outputBuffer, override.embeddings, override.byteOffset);
388
- }
389
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
390
- }
391
-
392
- if (isGpuBufferInstance(override.embeddings)) {
393
- const overrideTensor = createTensor(
394
- override.embeddings,
395
- 'f32',
396
- [override.prefixLength, hiddenSize],
397
- 'prefill_embedding_override_f32'
398
- );
399
- assertImplicitDtypeTransitionAllowed({
400
- executionPolicies,
401
- fromDtype: 'f32',
402
- toDtype: 'f16',
403
- op: 'prefill_embedding_override',
404
- detail: 'Prefix embedding override would pack GPU-provided f32 features into an f16 activation buffer.',
405
- });
406
- const castedOverride = await castF32ToF16(overrideTensor);
407
- try {
408
- encoder.copyBufferToBuffer(
409
- castedOverride.buffer,
410
- 0,
411
- outputBuffer,
412
- targetByteOffset,
413
- castedOverride.buffer.size
414
- );
415
- device.queue.submit([encoder.finish()]);
416
- } finally {
417
- releaseBuffer(castedOverride.buffer);
418
- }
419
- } else {
420
- assertImplicitDtypeTransitionAllowed({
421
- executionPolicies,
422
- fromDtype: 'f32',
423
- toDtype: 'f16',
424
- op: 'prefill_embedding_override',
425
- detail: 'Prefix embedding override would pack CPU-provided f32 features into an f16 activation buffer.',
426
- });
427
- const packedOverride = f32ToF16Array(override.embeddings);
428
- device.queue.submit([encoder.finish()]);
429
- uploadData(outputBuffer, packedOverride, targetByteOffset);
430
- }
431
- return createTensor(outputBuffer, baseTensor.dtype, [...baseTensor.shape], 'prefill_embedding_override');
432
- } catch (error) {
433
- releaseBuffer(outputBuffer);
434
- throw error;
435
- }
436
- }
437
-
438
- /**
439
- * Resolve display text for a token ID sequence.
440
- *
441
- * Preference order:
442
- * 1. renderTokenText (custom primary renderer) — if provided and returns non-empty string.
443
- * 2. tokenizer.decode (default primary) — if renderTokenText is not provided.
444
- * 3. renderFallbackTokenText (custom fallback) — if provided and returns non-empty string,
445
- * unless primary returned empty string and fallback looks like a special token (<...>),
446
- * in which case empty string is preserved to keep skip-special behavior deterministic.
447
- * 4. tokenizer.decode with skipSpecialTokens=false (default fallback) — same special-token guard.
448
- * 5. fallbackText — static fallback string (default '?').
449
- */
450
- function resolveTokenText(tokenizer, tokenIds, fallbackText = '?', renderTokenText, renderFallbackTokenText) {
451
- const renderPrimary = typeof renderTokenText === 'function'
452
- ? renderTokenText
453
- : (ids) => tokenizer?.decode?.(ids);
454
- const renderFallback = typeof renderFallbackTokenText === 'function'
455
- ? renderFallbackTokenText
456
- : (ids) => tokenizer?.decode?.(ids, false);
457
-
458
- const primaryText = renderPrimary(tokenIds);
459
- if (typeof primaryText === 'string' && primaryText.length > 0) {
460
- return primaryText;
461
- }
462
-
463
- const fallback = renderFallback(tokenIds);
464
- if (typeof fallback === 'string' && fallback.length > 0) {
465
- // Keep skip-special behavior deterministic: if primary decoding filtered this
466
- // token to empty, do not reintroduce obvious special-token text via fallback.
467
- if (
468
- primaryText === ''
469
- && /^<[^>\n]{1,80}>$/.test(fallback.trim())
470
- ) {
471
- return '';
472
- }
473
- return fallback;
474
- }
475
-
476
- return fallbackText;
477
- }
478
-
479
- function usesReplayPrefillDecode(state) {
480
- return state?.modelConfig?.decodeStrategy === 'replay_prefill';
481
- }
482
-
483
- export function shouldDisablePrefillCommandBatching(state, opts, multimodalBidirectionalSpan) {
484
- if (
485
- opts?.disableCommandBatching === true
486
- || opts?.debug === true
487
- || (Array.isArray(opts?.debugLayers) && opts.debugLayers.length > 0)
488
- ) {
489
- return true;
490
- }
491
- if (state?.kvCache?.layout === 'bdpa_paged') {
492
- return true;
493
- }
494
- if (multimodalBidirectionalSpan == null) {
495
- return false;
496
- }
497
- if (state?.kvCache?.hasGPUCache?.() !== true) {
498
- return false;
499
- }
500
- // WORKAROUND: recorded prefill with live KV-cache writes regresses Gemma 4
501
- // multimodal logits on the first generated token. Keep this lane fail-closed
502
- // until the recorder/cache interaction is numerically matched to the direct path.
503
- return true;
504
- }
505
-
506
- function assertIncrementalDecodeSupport(state, operation) {
507
- if (!usesReplayPrefillDecode(state)) {
508
- return;
509
- }
510
- throw new Error(
511
- `[Pipeline] ${operation} is not supported for models that require replay-prefill decode. ` +
512
- 'Incremental KV-cache decode and prefix snapshots stay disabled when the model config does not resolve ' +
513
- 'explicit layerTypes for mixed-geometry/shared-KV decode.'
514
- );
515
- }
516
-
517
- function summarizeExecutionPlan(plan) {
518
- if (!plan) {
519
- return null;
520
- }
521
- if (typeof plan !== 'object') {
522
- log.warn('Pipeline', `summarizeExecutionPlan: expected object, got ${typeof plan}`);
523
- return null;
524
- }
525
- if (typeof plan.id !== 'string') {
526
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "id"');
527
- }
528
- if (typeof plan.activationDtype !== 'string') {
529
- log.warn('Pipeline', 'summarizeExecutionPlan: plan is missing required string property "activationDtype"');
530
- }
531
- return {
532
- id: plan.id,
533
- kernelPathId: plan.kernelPathId ?? null,
534
- kernelPathSource: plan.kernelPathSource ?? 'none',
535
- activationDtype: plan.activationDtype,
536
- readbackInterval: plan.readbackInterval ?? null,
537
- batchSize: plan.defaultBatchSize,
538
- stopCheckMode: plan.defaultStopCheckMode,
539
- ringTokens: plan.ringTokens ?? null,
540
- ringStop: plan.ringStop ?? null,
541
- ringStaging: plan.ringStaging ?? null,
542
- };
543
- }
544
-
545
- export function shouldRetryWithFinitenessFallback(error) {
546
- if (error?.name === 'FinitenessError') {
547
- return true;
548
- }
549
- const message = typeof error?.message === 'string'
550
- ? error.message
551
- : (typeof error === 'string' ? error : '');
552
- if (!message.startsWith('[Sampling]')) {
553
- return false;
554
- }
555
- return message.includes('no finite candidate logits after masking the pad token')
556
- || message.includes('Softmax produced no finite candidate probabilities');
557
- }
558
-
559
- function createUnhandledFinitenessPolicyError(state, contextLabel, error) {
560
- const activePlan = resolveActiveExecutionPlan(state);
561
- const wrapped = new Error(
562
- `[Pipeline] ${contextLabel}: finiteness guard triggered for kernelPath ` +
563
- `"${activePlan.kernelPathId ?? 'none'}" under fail-fast policy. ` +
564
- 'Resolve the unstable path with an explicit capability-aware execution override, ' +
565
- 'or opt into alternate-plan recovery with ' +
566
- 'runtime.inference.compute.rangeAwareSelectiveWidening.onTrigger="fallback-plan".',
567
- error instanceof Error ? { cause: error } : undefined
568
- );
569
- wrapped.name = error?.name === 'FinitenessError' ? error.name : 'FinitenessError';
570
- return wrapped;
571
- }
572
-
573
- function assertResolvedKVDtype(kvDtype, contextLabel) {
574
- if (kvDtype === 'f16' || kvDtype === 'f32') {
575
- return kvDtype;
576
- }
577
- throw new Error(
578
- `[Pipeline] ${contextLabel}: expected execution-plan kvDtype to resolve to "f16" or "f32", ` +
579
- `got ${JSON.stringify(kvDtype)}.`
580
- );
581
- }
582
-
583
- function resolveTargetPlanKVDtype(plan, contextLabel) {
584
- return assertResolvedKVDtype(
585
- plan?.kernelPath?.kvDtype ?? plan?.activationDtype ?? null,
586
- contextLabel
587
- );
588
- }
589
-
590
- function resolveCurrentKVCacheDtype(state, plan, contextLabel) {
591
- return assertResolvedKVDtype(
592
- state?.kvCache?.kvDtype
593
- ?? plan?.kernelPath?.kvDtype
594
- ?? state?.runtimeConfig?.inference?.session?.kvcache?.kvDtype
595
- ?? plan?.activationDtype
596
- ?? null,
597
- contextLabel
598
- );
599
- }
600
-
601
- function cloneRuntimeInferenceWithKVDtype(state, kvDtype) {
602
- const runtimeInference = state?.runtimeConfig?.inference;
603
- if (!runtimeInference?.session?.kvcache) {
604
- throw new Error(
605
- '[Pipeline] runtime.inference.session.kvcache is required for finiteness fallback KV-cache recovery.'
606
- );
607
- }
608
- return mergeRuntimeValues(runtimeInference, {
609
- session: {
610
- kvcache: {
611
- kvDtype,
612
- },
613
- },
614
- });
615
- }
616
121
 
617
122
  export class PipelineGenerator {
618
123
 
@@ -812,14 +317,36 @@ export class PipelineGenerator {
812
317
  _sampleNextTokenFromLogits(logits, generatedIds, opts) {
813
318
  const sampledLogits = Float32Array.from(logits);
814
319
  applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
320
+ // Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
321
+ // implement grammar/schema-constrained decoding. The hook receives the
322
+ // mutable logit buffer (after repetition penalty) plus the running token
323
+ // sequence so it can track parse state across decode steps.
324
+ if (typeof opts?.logitMaskFn === "function") {
325
+ try {
326
+ opts.logitMaskFn(sampledLogits, {
327
+ generatedIds,
328
+ tokenizer: this.#state.tokenizer ?? null,
329
+ vocabSize: this.#state.modelConfig?.vocabSize ?? sampledLogits.length,
330
+ });
331
+ } catch (maskError) {
332
+ log.warn("Pipeline", `logitMaskFn threw; continuing without mask: ${maskError}`);
333
+ }
334
+ }
815
335
  const padTokenId = this.#state.tokenizer?.getSpecialTokens?.()?.pad;
816
- return sample(sampledLogits, {
336
+ const tokenId = sample(sampledLogits, {
817
337
  temperature: opts.temperature,
818
338
  topP: opts.topP,
819
339
  topK: opts.topK,
820
340
  padTokenId,
821
341
  seed: opts.seed,
822
342
  });
343
+ if (typeof opts.onLogits === 'function') {
344
+ opts.onLogits(sampledLogits, {
345
+ tokenId,
346
+ inputTokenCount: Array.isArray(generatedIds) ? generatedIds.length : null,
347
+ });
348
+ }
349
+ return tokenId;
823
350
  }
824
351
 
825
352
  async _prefillPromptToLogits(prompt, opts, contextLabel) {
@@ -880,6 +407,11 @@ export class PipelineGenerator {
880
407
  return this._matchesStopSequence(generatedIds, runtime.stopSequenceStart, opts.stopSequences);
881
408
  }
882
409
 
410
+ _recordStopReason(reason, tokenId = null) {
411
+ this.#state.stats.stopReason = reason;
412
+ this.#state.stats.stopTokenId = Number.isInteger(tokenId) ? tokenId : null;
413
+ }
414
+
883
415
  async *_generateTokensInternal(prompt, options = {}, mode = 'text') {
884
416
  if (!this.#state.isLoaded) throw new Error('Model not loaded');
885
417
  if (this.#state.isGenerating) throw new Error('Generation already in progress');
@@ -893,6 +425,8 @@ export class PipelineGenerator {
893
425
  this.#state.stats.decodeRecordMs = 0;
894
426
  this.#state.stats.decodeSubmitWaitMs = 0;
895
427
  this.#state.stats.decodeReadbackWaitMs = 0;
428
+ this.#state.stats.prefillRecordMs = 0;
429
+ this.#state.stats.prefillSubmitWaitMs = 0;
896
430
  this.#state.stats.singleTokenSubmitWaitMs = 0;
897
431
  this.#state.stats.singleTokenReadbackWaitMs = 0;
898
432
  this.#state.stats.singleTokenOrchestrationMs = 0;
@@ -904,10 +438,14 @@ export class PipelineGenerator {
904
438
  this.#state.stats.plePreparedTokenCacheBytes = 0;
905
439
  this.#state.stats.decodeMode = null;
906
440
  this.#state.stats.batchGuardReason = null;
441
+ this.#state.stats.stopReason = null;
442
+ this.#state.stats.stopTokenId = null;
907
443
  this.#state.stats.ttftMs = 0;
908
444
  const startTime = performance.now();
909
445
 
910
446
  const opts = resolveGenerateOptions(this.#state, options);
447
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
448
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
911
449
  // Validate and normalize sampling parameters through single source of truth
912
450
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
913
451
  opts.temperature = samplingConfig.temperature;
@@ -916,7 +454,8 @@ export class PipelineGenerator {
916
454
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
917
455
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
918
456
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
919
- if (diagnosticsEnabled) {
457
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
458
+ if (diagnosticsEnabled || tsirFixtureCfg) {
920
459
  const captureConfig = {
921
460
  ...createDefaultCaptureConfig(),
922
461
  enabled: true,
@@ -925,13 +464,20 @@ export class PipelineGenerator {
925
464
  };
926
465
  validateCaptureConfig(captureConfig);
927
466
  this.#state.operatorDiagnostics = {
928
- enabled: true,
467
+ enabled: diagnosticsEnabled === true,
929
468
  captureConfig,
930
- emitter: new OperatorEventEmitter({
469
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
931
470
  modelHash: this.#state.manifest?.modelId ?? null,
932
471
  runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
933
472
  executionPlanHash: opts.executionPlan?.id ?? null,
934
- }),
473
+ }) : null,
474
+ tsirFixture: tsirFixtureCfg ? {
475
+ dir: tsirFixtureCfg.dir,
476
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
477
+ ? tsirFixtureCfg.layerFilter
478
+ : null,
479
+ records: [],
480
+ } : null,
935
481
  };
936
482
  }
937
483
  const activePlan = opts.executionPlan ?? resolveActiveExecutionPlan(this.#state);
@@ -1051,6 +597,7 @@ export class PipelineGenerator {
1051
597
  yield* emitToken(this, firstToken, decodeToken);
1052
598
 
1053
599
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
600
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1054
601
  this.#state.stats.decodeTimeMs = 0;
1055
602
  this.#state.stats.tokensGenerated = 1;
1056
603
  this.#state.stats.decodeTokens = 1;
@@ -1277,6 +824,31 @@ export class PipelineGenerator {
1277
824
  // Generation Public API
1278
825
  // ==========================================================================
1279
826
 
827
+ /**
828
+ * Truncate the KV cache back to `seqLen` tokens and set `currentSeqLen` to
829
+ * match. Intended for "prefix-reuse" workflows where a caller wants to run
830
+ * several decodes that share a common prompt prefix: prefill once with the
831
+ * shared prefix, decode the first tail, then `resetToSeqLen(prefixLen)` to
832
+ * drop the tail's KV entries and reuse the prefix KV for the next tail.
833
+ *
834
+ * Only valid when no decode is in progress.
835
+ */
836
+ resetToSeqLen(seqLen) {
837
+ if (this.#state.isGenerating) {
838
+ throw new Error('InferencePipeline.resetToSeqLen: cannot reset while generation is in progress');
839
+ }
840
+ const target = Math.max(0, Math.floor(Number(seqLen) || 0));
841
+ if (!Number.isFinite(target)) {
842
+ throw new Error('InferencePipeline.resetToSeqLen: seqLen must be a finite non-negative integer');
843
+ }
844
+ if (target > this.#state.currentSeqLen) {
845
+ throw new Error(
846
+ `InferencePipeline.resetToSeqLen: target ${target} exceeds currentSeqLen ${this.#state.currentSeqLen}`
847
+ );
848
+ }
849
+ this.#state.kvCache?.truncate?.(target);
850
+ this.#state.currentSeqLen = target;
851
+ }
1280
852
 
1281
853
  async *generate(prompt, options = {}) {
1282
854
  yield* this._generateTokensInternal(prompt, options, 'text');
@@ -1299,6 +871,8 @@ export class PipelineGenerator {
1299
871
  this.#state.stats.decodeRecordMs = 0;
1300
872
  this.#state.stats.decodeSubmitWaitMs = 0;
1301
873
  this.#state.stats.decodeReadbackWaitMs = 0;
874
+ this.#state.stats.prefillRecordMs = 0;
875
+ this.#state.stats.prefillSubmitWaitMs = 0;
1302
876
  this.#state.stats.singleTokenSubmitWaitMs = 0;
1303
877
  this.#state.stats.singleTokenReadbackWaitMs = 0;
1304
878
  this.#state.stats.singleTokenOrchestrationMs = 0;
@@ -1310,9 +884,12 @@ export class PipelineGenerator {
1310
884
  this.#state.stats.plePreparedTokenCacheBytes = 0;
1311
885
  this.#state.stats.decodeMode = null;
1312
886
  this.#state.stats.batchGuardReason = null;
887
+ this.#state.stats.stopReason = null;
888
+ this.#state.stats.stopTokenId = null;
1313
889
  this.#state.stats.ttftMs = 0;
1314
890
  const startTime = performance.now();
1315
891
  const opts = resolveGenerateOptions(this.#state, options);
892
+ opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
1316
893
  // Validate and normalize sampling parameters through single source of truth
1317
894
  const samplingConfig = resolveSamplingConfig(options, this.#state.runtimeConfig);
1318
895
  opts.temperature = samplingConfig.temperature;
@@ -1321,7 +898,8 @@ export class PipelineGenerator {
1321
898
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
1322
899
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
1323
900
  || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
1324
- if (diagnosticsEnabled) {
901
+ const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
902
+ if (diagnosticsEnabled || tsirFixtureCfg) {
1325
903
  const captureConfig = {
1326
904
  ...createDefaultCaptureConfig(),
1327
905
  enabled: true,
@@ -1330,13 +908,20 @@ export class PipelineGenerator {
1330
908
  };
1331
909
  validateCaptureConfig(captureConfig);
1332
910
  this.#state.operatorDiagnostics = {
1333
- enabled: true,
911
+ enabled: diagnosticsEnabled === true,
1334
912
  captureConfig,
1335
- emitter: new OperatorEventEmitter({
913
+ emitter: diagnosticsEnabled ? new OperatorEventEmitter({
1336
914
  modelHash: this.#state.manifest?.modelId ?? null,
1337
915
  runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
1338
916
  executionPlanHash: opts.executionPlan?.id ?? null,
1339
- }),
917
+ }) : null,
918
+ tsirFixture: tsirFixtureCfg ? {
919
+ dir: tsirFixtureCfg.dir,
920
+ layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
921
+ ? tsirFixtureCfg.layerFilter
922
+ : null,
923
+ records: [],
924
+ } : null,
1340
925
  };
1341
926
  }
1342
927
 
@@ -1388,6 +973,7 @@ export class PipelineGenerator {
1388
973
  tokenIds.push(tokenId);
1389
974
  }
1390
975
  } else {
976
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1391
977
  this.#state.stats.decodeTimeMs = 0;
1392
978
  this.#state.stats.tokensGenerated = 1;
1393
979
  this.#state.stats.decodeTokens = 1;
@@ -1726,6 +1312,7 @@ export class PipelineGenerator {
1726
1312
  if (options.onToken) options.onToken(firstToken, firstText);
1727
1313
 
1728
1314
  if (this._shouldStopAfterAppendedToken(generatedIds, firstToken, opts, decodeRuntime)) {
1315
+ this._recordStopReason('stop-token-or-sequence', firstToken);
1729
1316
  this.#state.stats.decodeTimeMs = 0;
1730
1317
  this.#state.stats.tokensGenerated = 1;
1731
1318
  this.#state.stats.decodeTokens = 1;
@@ -1852,7 +1439,10 @@ export class PipelineGenerator {
1852
1439
  }
1853
1440
 
1854
1441
  while (tokensGenerated < opts.maxTokens) {
1855
- if (options.signal?.aborted) break;
1442
+ if (options.signal?.aborted) {
1443
+ this._recordStopReason('aborted');
1444
+ break;
1445
+ }
1856
1446
  if (this._hasFinitenessFallbackWindow() && useBatchPath) {
1857
1447
  useBatchPath = false;
1858
1448
  }
@@ -1901,9 +1491,15 @@ export class PipelineGenerator {
1901
1491
  if (batchTokens.length > 0 && options.onBatch) options.onBatch(batchTokens);
1902
1492
  if (opts.stopSequences.length > 0) {
1903
1493
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1904
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1494
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1495
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
1496
+ break;
1497
+ }
1498
+ }
1499
+ if (hitStop) {
1500
+ this._recordStopReason('stop-token');
1501
+ break;
1905
1502
  }
1906
- if (hitStop) break;
1907
1503
  if (shouldDisableBatchDecodeAfterShortBatch({
1908
1504
  hitStop,
1909
1505
  actualCount: batchResult.actualCount,
@@ -1941,7 +1537,10 @@ export class PipelineGenerator {
1941
1537
  if (options.onToken) options.onToken(nextToken, tokenText);
1942
1538
  }
1943
1539
  this._consumeFinitenessFallbackToken(opts);
1944
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
1540
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
1541
+ this._recordStopReason('stop-token', nextToken);
1542
+ break;
1543
+ }
1945
1544
  }
1946
1545
  } else if (opts.speculation?.mode === 'self') {
1947
1546
  // Self-speculation: decode one base token plus a configurable burst of
@@ -1979,11 +1578,17 @@ export class PipelineGenerator {
1979
1578
  }
1980
1579
  this._consumeFinitenessFallbackToken(opts);
1981
1580
 
1982
- if (isStopToken(baseToken, stopTokenIds, eosToken)) break;
1581
+ if (isStopToken(baseToken, stopTokenIds, eosToken)) {
1582
+ this._recordStopReason('stop-token', baseToken);
1583
+ break;
1584
+ }
1983
1585
  if (tokensGenerated >= opts.maxTokens) break;
1984
1586
  if (opts.stopSequences.length > 0) {
1985
1587
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
1986
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1588
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1589
+ this._recordStopReason('stop-sequence', baseToken);
1590
+ break;
1591
+ }
1987
1592
  }
1988
1593
 
1989
1594
  for (let specIndex = 0; specIndex < speculativeBurstTokens; specIndex += 1) {
@@ -2022,19 +1627,27 @@ export class PipelineGenerator {
2022
1627
  }
2023
1628
 
2024
1629
  if (isStopToken(specToken, stopTokenIds, eosToken)) {
1630
+ this._recordStopReason('stop-token', specToken);
2025
1631
  break;
2026
1632
  }
2027
1633
  if (opts.stopSequences.length > 0) {
2028
1634
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2029
1635
  if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1636
+ this._recordStopReason('stop-sequence', specToken);
2030
1637
  break;
2031
1638
  }
2032
1639
  }
2033
1640
  }
2034
- if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) break;
1641
+ if (isStopToken(generatedIds[generatedIds.length - 1], stopTokenIds, eosToken)) {
1642
+ this._recordStopReason('stop-token', generatedIds[generatedIds.length - 1]);
1643
+ break;
1644
+ }
2035
1645
  if (opts.stopSequences.length > 0) {
2036
1646
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2037
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1647
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1648
+ this._recordStopReason('stop-sequence', generatedIds[generatedIds.length - 1] ?? null);
1649
+ break;
1650
+ }
2038
1651
  }
2039
1652
  } else {
2040
1653
  const tokenStart = performance.now();
@@ -2088,14 +1701,23 @@ export class PipelineGenerator {
2088
1701
  log.debug('Decode', `#${tokensGenerated} "${tokenText}" ${tokenTime.toFixed(0)}ms (${tokPerSec.toFixed(2)} tok/s avg)`);
2089
1702
  }
2090
1703
 
2091
- if (isStopToken(nextToken, stopTokenIds, eosToken)) break;
1704
+ if (isStopToken(nextToken, stopTokenIds, eosToken)) {
1705
+ this._recordStopReason('stop-token', nextToken);
1706
+ break;
1707
+ }
2092
1708
  if (opts.stopSequences.length > 0) {
2093
1709
  const fullText = this.#state.tokenizer.decode(generatedIds.slice(stopSequenceStart), false);
2094
- if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) break;
1710
+ if (opts.stopSequences.some((seq) => fullText.endsWith(seq))) {
1711
+ this._recordStopReason('stop-sequence', nextToken);
1712
+ break;
1713
+ }
2095
1714
  }
2096
1715
  }
2097
1716
  }
2098
1717
 
1718
+ if (!this.#state.stats.stopReason) {
1719
+ this._recordStopReason(tokensGenerated >= opts.maxTokens ? 'max-tokens' : 'completed');
1720
+ }
2099
1721
  this.#state.stats.decodeTimeMs = performance.now() - decodeStart;
2100
1722
  this.#state.stats.tokensGenerated = tokensGenerated;
2101
1723
  this.#state.stats.decodeTokens = tokensGenerated;
@@ -2321,16 +1943,35 @@ export class PipelineGenerator {
2321
1943
  // Chunked recorder submission: submit every N layers to release tracked intermediate
2322
1944
  // buffers, preventing unbounded memory growth during large prefills. Critical for
2323
1945
  // replay_prefill models where each decode step re-runs a prefill-style layer pass.
2324
- const PREFILL_RECORDER_CHUNK_LAYERS = 4;
1946
+ const prefillRecorderChunkLayers = resolvePrefillRecorderChunkLayers({
1947
+ hasGpuSplitPerLayerInputs: context.perLayerInputsSession?.materialization === 'gpu_split_tables',
1948
+ numTokens,
1949
+ });
2325
1950
 
2326
1951
  let currentHiddenBuffer = hiddenStates.buffer;
1952
+ let prefillRecordMs = 0;
1953
+ let prefillSubmitWaitMs = 0;
2327
1954
  try {
2328
1955
  for (let l = 0; l < config.numLayers; l++) {
1956
+ // Per-layer hard cancellation: when the caller's AbortSignal aborts,
1957
+ // exit the prefill loop between layer dispatches rather than
1958
+ // continuing to burn GPU on superseded work. Granularity is one
1959
+ // layer (~5-50ms), which is the fastest cancel granularity WebGPU
1960
+ // exposes today.
1961
+ if (opts?.signal?.aborted) {
1962
+ const reason = typeof opts.signal.reason === "string" ? opts.signal.reason : "Doppler: prefill aborted";
1963
+ const err = new Error(reason);
1964
+ err.name = "AbortError";
1965
+ err.code = "ABORT_ERR";
1966
+ throw err;
1967
+ }
2329
1968
  context.recorder = currentRecorder;
2330
1969
  context.perLayerInputBuffer = perLayerInputs?.[l] ?? null;
2331
1970
 
2332
1971
  const prevBuffer = currentHiddenBuffer;
1972
+ const layerRecordStart = performance.now();
2333
1973
  const layerOutput = await processLayer(l, currentHiddenBuffer, numTokens, true, context);
1974
+ prefillRecordMs += performance.now() - layerRecordStart;
2334
1975
  if (!isGpuBufferInstance(layerOutput)) throw new Error('Expected GPUBuffer from processLayer');
2335
1976
  currentHiddenBuffer = layerOutput;
2336
1977
  releasePerLayerInputBuffer(
@@ -2348,7 +1989,7 @@ export class PipelineGenerator {
2348
1989
  const isChunkBoundary = !isCheckpoint
2349
1990
  && currentRecorder
2350
1991
  && l < config.numLayers - 1
2351
- && (l + 1) % PREFILL_RECORDER_CHUNK_LAYERS === 0;
1992
+ && (l + 1) % prefillRecorderChunkLayers === 0;
2352
1993
 
2353
1994
  if (isCheckpoint && currentRecorder) {
2354
1995
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
@@ -2408,8 +2049,25 @@ export class PipelineGenerator {
2408
2049
  currentRecorder,
2409
2050
  'prefill_chunk_carry'
2410
2051
  );
2411
- await currentRecorder.submitAndWait();
2412
- await recordProfile(currentRecorder);
2052
+ // Chunk boundary exists only to bound intermediate buffer lifetime.
2053
+ // When the runtime opts into async chunk submit AND profile timings
2054
+ // are not being collected, skip the CPU-GPU wait: queue order is
2055
+ // preserved across submits and deferred cleanup still releases
2056
+ // tracked buffers when GPU work completes. Profile runs keep the
2057
+ // sync path because resolveProfileTimings requires the submitted
2058
+ // work to be done.
2059
+ const manifestSubmit = this.#state.modelConfig?.sessionSettings?.prefillChunkSubmitMode;
2060
+ const chunkSubmitMode = (manifestSubmit !== null && manifestSubmit !== undefined)
2061
+ ? manifestSubmit
2062
+ : (this.#state.runtimeConfig?.inference?.session?.prefillChunkSubmitMode ?? 'sync');
2063
+ const chunkSubmitStart = performance.now();
2064
+ if (chunkSubmitMode === 'async' && !opts.profile) {
2065
+ currentRecorder.submit({ cleanup: 'deferred' });
2066
+ } else {
2067
+ await currentRecorder.submitAndWait();
2068
+ await recordProfile(currentRecorder);
2069
+ }
2070
+ prefillSubmitWaitMs += performance.now() - chunkSubmitStart;
2413
2071
  currentRecorder = createRecorder('prefill-chunk');
2414
2072
  }
2415
2073
  }
@@ -2426,8 +2084,26 @@ export class PipelineGenerator {
2426
2084
  }
2427
2085
  }
2428
2086
  releaseSharedAttentionState(context.sharedAttentionState, currentRecorder);
2087
+ this.#state.stats.prefillRecordMs = (this.#state.stats.prefillRecordMs ?? 0) + prefillRecordMs;
2088
+ this.#state.stats.prefillSubmitWaitMs = (this.#state.stats.prefillSubmitWaitMs ?? 0) + prefillSubmitWaitMs;
2429
2089
  }
2430
2090
 
2091
+ {
2092
+ const tsirFixtureCfg = this.#state.operatorDiagnostics?.tsirFixture ?? null;
2093
+ if (tsirFixtureCfg && Array.isArray(tsirFixtureCfg.pendingReads) && tsirFixtureCfg.pendingReads.length > 0) {
2094
+ if (currentRecorder) {
2095
+ currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(
2096
+ currentHiddenBuffer,
2097
+ currentRecorder,
2098
+ 'prefill_tsir_drain_carry'
2099
+ );
2100
+ await currentRecorder.submitAndWait();
2101
+ await recordProfile(currentRecorder);
2102
+ currentRecorder = undefined;
2103
+ }
2104
+ await drainPendingTsirReads(tsirFixtureCfg);
2105
+ }
2106
+ }
2431
2107
  if (this.#state.finitenessBuffer) {
2432
2108
  if (currentRecorder) {
2433
2109
  currentHiddenBuffer = preserveBufferAcrossRecorderSubmit(