doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -38,6 +38,8 @@ import {
38
38
  safeToFixed,
39
39
  sampleTimingNumber,
40
40
  buildCanonicalTiming,
41
+ buildLoadTimingDiagnostics,
42
+ buildDecodeBottleneckDiagnostics,
41
43
  buildTimingDiagnostics,
42
44
  } from './browser-harness-suite-helpers.js';
43
45
  import {
@@ -47,6 +49,10 @@ import {
47
49
  } from './browser-harness-model-helpers.js';
48
50
  import {
49
51
  resolveBenchmarkRunSettings,
52
+ normalizeDecodeRecordOpLabels,
53
+ normalizeUniformCacheStats,
54
+ buildDecodeRecordTopOps,
55
+ buildDecodeRecordTopOpGroups,
50
56
  runEmbeddingSemanticChecks,
51
57
  isCoherentOutput,
52
58
  runTextInference,
@@ -64,6 +70,21 @@ import { stableSortObject } from '../utils/stable-sort-object.js';
64
70
  const TRAINING_SUITE_MODULE_PATH = '../experimental/training/suite.js';
65
71
  let trainingSuiteModulePromise = null;
66
72
 
73
+ function resolvePipelineLoadTimings(pipeline) {
74
+ if (!pipeline || typeof pipeline.getStats !== 'function') {
75
+ return { loadTiming: null, pipelineLoadTiming: null };
76
+ }
77
+ try {
78
+ const stats = pipeline.getStats() ?? {};
79
+ return {
80
+ loadTiming: stats.loadTiming ?? null,
81
+ pipelineLoadTiming: stats.pipelineLoadTiming ?? null,
82
+ };
83
+ } catch {
84
+ return { loadTiming: null, pipelineLoadTiming: null };
85
+ }
86
+ }
87
+
67
88
  async function loadTrainingSuiteModule() {
68
89
  if (!trainingSuiteModulePromise) {
69
90
  trainingSuiteModulePromise = import(TRAINING_SUITE_MODULE_PATH);
@@ -125,6 +146,83 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
125
146
  }),
126
147
  });
127
148
 
149
+ function isPlainObject(value) {
150
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
151
+ }
152
+
153
+ function resolveExecutionPlanCadence(executionPlan) {
154
+ if (!isPlainObject(executionPlan)) {
155
+ return null;
156
+ }
157
+ const finalActivePlanId = typeof executionPlan.finalActivePlanId === 'string'
158
+ ? executionPlan.finalActivePlanId
159
+ : null;
160
+ for (const candidate of [executionPlan.primary, executionPlan.fallback]) {
161
+ if (!isPlainObject(candidate)) {
162
+ continue;
163
+ }
164
+ if (finalActivePlanId == null || candidate.id === finalActivePlanId) {
165
+ return candidate;
166
+ }
167
+ }
168
+ return isPlainObject(executionPlan.primary) ? executionPlan.primary : null;
169
+ }
170
+
171
+ function resolveDecodeCadence(runtimeConfig, executionPlan = null) {
172
+ const inference = runtimeConfig?.inference;
173
+ const batching = inference?.batching;
174
+ const session = inference?.session;
175
+ const decodeLoop = session?.decodeLoop;
176
+ if (!isPlainObject(batching) || !isPlainObject(decodeLoop)) {
177
+ return null;
178
+ }
179
+ const executionPlanCadence = resolveExecutionPlanCadence(executionPlan);
180
+ const batchSize = executionPlanCadence?.batchSize ?? decodeLoop.batchSize ?? batching.batchSize ?? null;
181
+ const readbackInterval = executionPlanCadence?.readbackInterval ?? decodeLoop.readbackInterval ?? batching.readbackInterval ?? null;
182
+ return {
183
+ batchSize,
184
+ readbackInterval,
185
+ stopCheckMode: executionPlanCadence?.stopCheckMode ?? decodeLoop.stopCheckMode ?? batching.stopCheckMode ?? null,
186
+ readbackMode: executionPlanCadence?.readbackMode ?? decodeLoop.readbackMode ?? batching.readbackMode ?? null,
187
+ disableCommandBatching: executionPlanCadence?.disableCommandBatching ?? decodeLoop.disableCommandBatching ?? null,
188
+ disableMultiTokenDecode: inference?.generation?.disableMultiTokenDecode === true,
189
+ speculationMode: session?.speculation?.mode ?? null,
190
+ tokensPerReadback: Number.isFinite(batchSize) && Number.isFinite(readbackInterval)
191
+ ? batchSize * readbackInterval
192
+ : null,
193
+ runtimeMirror: {
194
+ batching: {
195
+ batchSize: batching.batchSize ?? null,
196
+ readbackInterval: batching.readbackInterval ?? null,
197
+ stopCheckMode: batching.stopCheckMode ?? null,
198
+ readbackMode: batching.readbackMode ?? null,
199
+ },
200
+ decodeLoop: {
201
+ batchSize: decodeLoop.batchSize ?? null,
202
+ readbackInterval: decodeLoop.readbackInterval ?? null,
203
+ stopCheckMode: decodeLoop.stopCheckMode ?? null,
204
+ readbackMode: decodeLoop.readbackMode ?? null,
205
+ ringTokens: decodeLoop.ringTokens ?? null,
206
+ ringStop: decodeLoop.ringStop ?? null,
207
+ ringStaging: decodeLoop.ringStaging ?? null,
208
+ },
209
+ },
210
+ executionPlan: executionPlanCadence
211
+ ? {
212
+ id: executionPlanCadence.id ?? null,
213
+ batchSize: executionPlanCadence.batchSize ?? null,
214
+ readbackInterval: executionPlanCadence.readbackInterval ?? null,
215
+ stopCheckMode: executionPlanCadence.stopCheckMode ?? null,
216
+ readbackMode: executionPlanCadence.readbackMode ?? null,
217
+ disableCommandBatching: executionPlanCadence.disableCommandBatching ?? null,
218
+ ringTokens: executionPlanCadence.ringTokens ?? null,
219
+ ringStop: executionPlanCadence.ringStop ?? null,
220
+ ringStaging: executionPlanCadence.ringStaging ?? null,
221
+ }
222
+ : null,
223
+ };
224
+ }
225
+
128
226
  export function getBrowserSupportedSuites() {
129
227
  return [...BROWSER_WORKLOAD_SET];
130
228
  }
@@ -566,6 +664,7 @@ async function runInferenceSuite(options = {}) {
566
664
  firstResponseMs: safeToFixed(safeModelLoadMs + run.phase.ttftMs),
567
665
  prefillMs: safeToFixed(run.phase.prefillMs),
568
666
  decodeMs: safeToFixed(run.phase.decodeMs),
667
+ wallRunMs: safeToFixed(run.phase.wallMs ?? run.durationMs),
569
668
  prefillTokens: Math.round(run.phase.prefillTokens),
570
669
  decodeTokens: Math.round(run.phase.decodeTokens),
571
670
  stopReason: run.phase.stopReason ?? null,
@@ -575,6 +674,7 @@ async function runInferenceSuite(options = {}) {
575
674
  decodeTokensPerSec: safeToFixed(run.phase.decodeTokensPerSec),
576
675
  modelLoadMs: safeModelLoadMs,
577
676
  gpu: run.phase.gpu,
677
+ batching: run.phase.batching ?? null,
578
678
  plePreparedTokenCache: run.phase.plePreparedTokenCache ?? null,
579
679
  prefillProfileSteps: run.phase.prefillProfileSteps,
580
680
  decodeProfileSteps: run.phase.decodeProfileSteps,
@@ -594,6 +694,12 @@ async function runInferenceSuite(options = {}) {
594
694
  const memoryStats = typeof harness.pipeline?.getMemoryStats === 'function'
595
695
  ? harness.pipeline.getMemoryStats()
596
696
  : null;
697
+ const loadTimings = resolvePipelineLoadTimings(harness.pipeline);
698
+ const loadDiagnostics = buildLoadTimingDiagnostics(
699
+ safeModelLoadMs,
700
+ loadTimings.loadTiming,
701
+ loadTimings.pipelineLoadTiming
702
+ );
597
703
  if (typeof harness.pipeline.unload === 'function' && !options.keepPipeline) {
598
704
  await harness.pipeline.unload();
599
705
  }
@@ -619,7 +725,16 @@ async function runInferenceSuite(options = {}) {
619
725
  const timingDiagnostics = buildTimingDiagnostics(timing, {
620
726
  source: 'doppler',
621
727
  prefillSemantics: 'internal_prefill_phase',
728
+ loadTiming: loadTimings.loadTiming,
729
+ pipelineLoadTiming: loadTimings.pipelineLoadTiming,
622
730
  });
731
+ const decodeBottleneck = buildDecodeBottleneckDiagnostics(metrics, timing);
732
+ const metricsWithTimingDiagnostics = decodeBottleneck
733
+ ? { ...metrics, decodeBottleneck }
734
+ : metrics;
735
+ if (decodeBottleneck) {
736
+ timingDiagnostics.decodeBottleneck = decodeBottleneck;
737
+ }
623
738
  const firstLoad = buildFirstLoadComposition({
624
739
  modelLoadMs: timing.modelLoadMs,
625
740
  firstTokenMs: timing.firstTokenMs,
@@ -627,7 +742,9 @@ async function runInferenceSuite(options = {}) {
627
742
  });
628
743
  const metricsWithContracts = buildSuiteContractMetrics(
629
744
  options.suiteName || 'inference',
630
- metrics,
745
+ loadDiagnostics
746
+ ? { ...metricsWithTimingDiagnostics, load: loadDiagnostics }
747
+ : metricsWithTimingDiagnostics,
631
748
  harness.manifest
632
749
  );
633
750
  return {
@@ -730,7 +847,7 @@ async function runBenchSuite(options = {}) {
730
847
  };
731
848
  }
732
849
 
733
- if (workloadType === 'diffusion') {
850
+ if (workloadType === 'diffusion' || workloadType === 'diffusion_gemma') {
734
851
  const diffusionBench = await runDiffusionSuite({
735
852
  ...options,
736
853
  command: 'bench',
@@ -757,7 +874,7 @@ async function runBenchSuite(options = {}) {
757
874
  results: benchResults,
758
875
  metrics: {
759
876
  ...(diffusionBench.metrics || {}),
760
- workloadType: 'diffusion',
877
+ workloadType,
761
878
  },
762
879
  };
763
880
  }
@@ -893,6 +1010,7 @@ async function runBenchSuite(options = {}) {
893
1010
  } else {
894
1011
  const tokensPerSec = [];
895
1012
  const durations = [];
1013
+ const phaseTotals = [];
896
1014
  const tokensGenerated = [];
897
1015
  const decodeMsPerToken = [];
898
1016
  const ttftMs = [];
@@ -906,19 +1024,41 @@ async function runBenchSuite(options = {}) {
906
1024
  const gpuPrefillMs = [];
907
1025
  const gpuDecodeMs = [];
908
1026
  const gpuDecodeRecordMs = [];
1027
+ const gpuDecodeRecordOps = [];
1028
+ const gpuDecodeRecordPasses = [];
1029
+ const gpuDecodeRecordMsPerOp = [];
1030
+ const gpuDecodeRecordMsPerPass = [];
1031
+ const gpuDecodeRecordPassesPerOp = [];
1032
+ const gpuDecodeRecordMsPerExecutedBatchToken = [];
1033
+ const gpuDecodeRecordOpsPerExecutedBatchToken = [];
1034
+ const gpuDecodeRecordPassesPerExecutedBatchToken = [];
1035
+ const gpuDecodeRecordOpLabels = {};
1036
+ let hasGpuDecodeRecordOpLabels = false;
909
1037
  const gpuDecodeSubmitWaitMs = [];
910
1038
  const gpuDecodeReadbackWaitMs = [];
1039
+ const gpuDecodeReadbackMapWaitMs = [];
1040
+ const gpuDecodeReadbackCleanupMs = [];
1041
+ const gpuDecodeReadbackCopyMs = [];
911
1042
  const gpuDecodeOrchestrationMs = [];
912
1043
  const gpuPrefillRecordMs = [];
913
1044
  const gpuPrefillSubmitWaitMs = [];
914
1045
  const singleTokenSubmitWaitMs = [];
915
1046
  const singleTokenReadbackWaitMs = [];
1047
+ const singleTokenReadbackMapWaitMs = [];
1048
+ const singleTokenReadbackCleanupMs = [];
1049
+ const singleTokenReadbackCopyMs = [];
916
1050
  const singleTokenOrchestrationMs = [];
917
1051
  const batchedForwardCalls = [];
918
1052
  const unbatchedForwardCalls = [];
919
1053
  const totalBatchedTimeMs = [];
920
1054
  const totalUnbatchedTimeMs = [];
921
1055
  const gpuSubmissions = [];
1056
+ const requestedBatchTokens = [];
1057
+ const effectiveBatchTokens = [];
1058
+ const executedBatchTokens = [];
1059
+ const resolvedBatchTokens = [];
1060
+ const maxBatchTokenCap = [];
1061
+ const batchClampCount = [];
922
1062
  const plePreparedTokenCacheHits = [];
923
1063
  const plePreparedTokenCacheMisses = [];
924
1064
  const plePreparedTokenCacheEntries = [];
@@ -926,8 +1066,13 @@ async function runBenchSuite(options = {}) {
926
1066
 
927
1067
  let generatedText = null;
928
1068
  let generatedPromptInput = null;
1069
+ let generatedReferenceTranscript = null;
1070
+ let lastPromptLabel = benchRun.promptLabel;
1071
+ let lastMaxTokens = benchRun.maxTokens;
929
1072
  let lastDecodeMode = null;
930
1073
  let lastBatchGuardReason = null;
1074
+ let lastExecutionPlan = null;
1075
+ let lastGpuUniformCache = null;
931
1076
  for (let i = 0; i < warmupRuns + timedRuns; i++) {
932
1077
  harness.pipeline.reset?.();
933
1078
  const run = await withHarnessPhase(
@@ -947,8 +1092,15 @@ async function runBenchSuite(options = {}) {
947
1092
  if (i === warmupRuns + timedRuns - 1) {
948
1093
  generatedText = run?.output ?? null;
949
1094
  generatedPromptInput = run?.promptInput ?? null;
1095
+ generatedReferenceTranscript = buildReferenceTranscriptSeed(run, {
1096
+ executionGraphHash: resolveExecutionGraphHash(harness.manifest),
1097
+ kvCache: run?.phase?.kvCache ?? null,
1098
+ });
1099
+ lastPromptLabel = run?.prompt ?? benchRun.promptLabel;
1100
+ lastMaxTokens = Number.isFinite(run?.maxTokens) ? run.maxTokens : benchRun.maxTokens;
950
1101
  lastDecodeMode = run?.phase?.decodeMode ?? null;
951
1102
  lastBatchGuardReason = run?.phase?.batchGuardReason ?? null;
1103
+ lastExecutionPlan = run?.phase?.executionPlan ?? null;
952
1104
  }
953
1105
  if (i >= warmupRuns) {
954
1106
  const phase = run?.phase ?? {};
@@ -958,6 +1110,7 @@ async function runBenchSuite(options = {}) {
958
1110
  const phasePlePreparedTokenCache = phase.plePreparedTokenCache;
959
1111
  tokensPerSec.push(run?.tokensPerSec);
960
1112
  durations.push(run?.durationMs);
1113
+ phaseTotals.push(phase.totalMs);
961
1114
  tokensGenerated.push(phaseTokens.length);
962
1115
  ttftMs.push(phase.ttftMs);
963
1116
  prefillMs.push(phase.prefillMs);
@@ -970,11 +1123,45 @@ async function runBenchSuite(options = {}) {
970
1123
  if (phase.decodeMs > 0 && phase.decodeTokens > 0) {
971
1124
  decodeMsPerToken.push(phase.decodeMs / phase.decodeTokens);
972
1125
  }
1126
+ const phaseGpuUniformCache = normalizeUniformCacheStats(phaseGpu?.uniformCache);
1127
+ if (phaseGpuUniformCache) {
1128
+ lastGpuUniformCache = phaseGpuUniformCache;
1129
+ }
973
1130
  if (Number.isFinite(phaseGpu?.prefillMs)) gpuPrefillMs.push(phaseGpu.prefillMs);
974
1131
  if (Number.isFinite(phaseGpu?.decodeMs)) gpuDecodeMs.push(phaseGpu.decodeMs);
975
1132
  if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
1133
+ if (Number.isFinite(phaseGpu?.decodeRecordOps)) gpuDecodeRecordOps.push(phaseGpu.decodeRecordOps);
1134
+ if (Number.isFinite(phaseGpu?.decodeRecordPasses)) gpuDecodeRecordPasses.push(phaseGpu.decodeRecordPasses);
1135
+ const phaseDecodeRecordOpLabels = normalizeDecodeRecordOpLabels(phaseGpu?.decodeRecordOpLabels);
1136
+ if (phaseDecodeRecordOpLabels) {
1137
+ hasGpuDecodeRecordOpLabels = true;
1138
+ for (const [label, count] of Object.entries(phaseDecodeRecordOpLabels)) {
1139
+ gpuDecodeRecordOpLabels[label] = (gpuDecodeRecordOpLabels[label] ?? 0) + count;
1140
+ }
1141
+ }
1142
+ if (Number.isFinite(phaseGpu?.decodeRecordMsPerOp)) {
1143
+ gpuDecodeRecordMsPerOp.push(phaseGpu.decodeRecordMsPerOp);
1144
+ }
1145
+ if (Number.isFinite(phaseGpu?.decodeRecordMsPerPass)) {
1146
+ gpuDecodeRecordMsPerPass.push(phaseGpu.decodeRecordMsPerPass);
1147
+ }
1148
+ if (Number.isFinite(phaseGpu?.decodeRecordPassesPerOp)) {
1149
+ gpuDecodeRecordPassesPerOp.push(phaseGpu.decodeRecordPassesPerOp);
1150
+ }
1151
+ if (Number.isFinite(phaseGpu?.decodeRecordMsPerExecutedBatchToken)) {
1152
+ gpuDecodeRecordMsPerExecutedBatchToken.push(phaseGpu.decodeRecordMsPerExecutedBatchToken);
1153
+ }
1154
+ if (Number.isFinite(phaseGpu?.decodeRecordOpsPerExecutedBatchToken)) {
1155
+ gpuDecodeRecordOpsPerExecutedBatchToken.push(phaseGpu.decodeRecordOpsPerExecutedBatchToken);
1156
+ }
1157
+ if (Number.isFinite(phaseGpu?.decodeRecordPassesPerExecutedBatchToken)) {
1158
+ gpuDecodeRecordPassesPerExecutedBatchToken.push(phaseGpu.decodeRecordPassesPerExecutedBatchToken);
1159
+ }
976
1160
  if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
977
1161
  if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
1162
+ if (Number.isFinite(phaseGpu?.decodeReadbackMapWaitMs)) gpuDecodeReadbackMapWaitMs.push(phaseGpu.decodeReadbackMapWaitMs);
1163
+ if (Number.isFinite(phaseGpu?.decodeReadbackCleanupMs)) gpuDecodeReadbackCleanupMs.push(phaseGpu.decodeReadbackCleanupMs);
1164
+ if (Number.isFinite(phaseGpu?.decodeReadbackCopyMs)) gpuDecodeReadbackCopyMs.push(phaseGpu.decodeReadbackCopyMs);
978
1165
  if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
979
1166
  if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
980
1167
  if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
@@ -982,12 +1169,21 @@ async function runBenchSuite(options = {}) {
982
1169
  }
983
1170
  if (Number.isFinite(phaseGpu?.singleTokenSubmitWaitMs)) singleTokenSubmitWaitMs.push(phaseGpu.singleTokenSubmitWaitMs);
984
1171
  if (Number.isFinite(phaseGpu?.singleTokenReadbackWaitMs)) singleTokenReadbackWaitMs.push(phaseGpu.singleTokenReadbackWaitMs);
1172
+ if (Number.isFinite(phaseGpu?.singleTokenReadbackMapWaitMs)) singleTokenReadbackMapWaitMs.push(phaseGpu.singleTokenReadbackMapWaitMs);
1173
+ if (Number.isFinite(phaseGpu?.singleTokenReadbackCleanupMs)) singleTokenReadbackCleanupMs.push(phaseGpu.singleTokenReadbackCleanupMs);
1174
+ if (Number.isFinite(phaseGpu?.singleTokenReadbackCopyMs)) singleTokenReadbackCopyMs.push(phaseGpu.singleTokenReadbackCopyMs);
985
1175
  if (Number.isFinite(phaseGpu?.singleTokenOrchestrationMs)) singleTokenOrchestrationMs.push(phaseGpu.singleTokenOrchestrationMs);
986
1176
  if (Number.isFinite(phaseBatching?.batchedForwardCalls)) batchedForwardCalls.push(phaseBatching.batchedForwardCalls);
987
1177
  if (Number.isFinite(phaseBatching?.unbatchedForwardCalls)) unbatchedForwardCalls.push(phaseBatching.unbatchedForwardCalls);
988
1178
  if (Number.isFinite(phaseBatching?.totalBatchedTimeMs)) totalBatchedTimeMs.push(phaseBatching.totalBatchedTimeMs);
989
1179
  if (Number.isFinite(phaseBatching?.totalUnbatchedTimeMs)) totalUnbatchedTimeMs.push(phaseBatching.totalUnbatchedTimeMs);
990
1180
  if (Number.isFinite(phaseBatching?.gpuSubmissions)) gpuSubmissions.push(phaseBatching.gpuSubmissions);
1181
+ if (Number.isFinite(phaseBatching?.requestedBatchTokens)) requestedBatchTokens.push(phaseBatching.requestedBatchTokens);
1182
+ if (Number.isFinite(phaseBatching?.effectiveBatchTokens)) effectiveBatchTokens.push(phaseBatching.effectiveBatchTokens);
1183
+ if (Number.isFinite(phaseBatching?.executedBatchTokens)) executedBatchTokens.push(phaseBatching.executedBatchTokens);
1184
+ if (Number.isFinite(phaseBatching?.resolvedBatchTokens)) resolvedBatchTokens.push(phaseBatching.resolvedBatchTokens);
1185
+ if (Number.isFinite(phaseBatching?.maxBatchTokenCap)) maxBatchTokenCap.push(phaseBatching.maxBatchTokenCap);
1186
+ if (Number.isFinite(phaseBatching?.batchClampCount)) batchClampCount.push(phaseBatching.batchClampCount);
991
1187
  if (Number.isFinite(phasePlePreparedTokenCache?.hits)) plePreparedTokenCacheHits.push(phasePlePreparedTokenCache.hits);
992
1188
  if (Number.isFinite(phasePlePreparedTokenCache?.misses)) plePreparedTokenCacheMisses.push(phasePlePreparedTokenCache.misses);
993
1189
  if (Number.isFinite(phasePlePreparedTokenCache?.entries)) plePreparedTokenCacheEntries.push(phasePlePreparedTokenCache.entries);
@@ -995,7 +1191,8 @@ async function runBenchSuite(options = {}) {
995
1191
  }
996
1192
  }
997
1193
 
998
- const totalMsStats = computeSampleStats(durations);
1194
+ const totalMsStats = computeSampleStats(phaseTotals);
1195
+ const wallRunMsStats = computeSampleStats(durations);
999
1196
  const tokensPerSecStats = computeSampleStats(tokensPerSec);
1000
1197
  const decodeTokensPerSecStats = computeSampleStats(decodeTokensPerSec);
1001
1198
  const prefillTokensPerSecStats = computeSampleStats(prefillTokensPerSec);
@@ -1007,23 +1204,74 @@ async function runBenchSuite(options = {}) {
1007
1204
  const tokensGeneratedStats = computeSampleStats(tokensGenerated);
1008
1205
  const prefillTokensStats = computeSampleStats(prefillTokens);
1009
1206
  const decodeTokensStats = computeSampleStats(decodeTokens);
1207
+ const gpuDecodeRecordOpsStats = computeSampleStats(gpuDecodeRecordOps);
1208
+ const gpuDecodeRecordPassesStats = computeSampleStats(gpuDecodeRecordPasses);
1209
+ const gpuDecodeRecordOpLabelSampleCount = gpuDecodeRecordOps.length > 0
1210
+ ? gpuDecodeRecordOps.length
1211
+ : 1;
1212
+ const gpuDecodeRecordMeanOpLabels = {};
1213
+ if (hasGpuDecodeRecordOpLabels) {
1214
+ for (const [label, count] of Object.entries(gpuDecodeRecordOpLabels)) {
1215
+ gpuDecodeRecordMeanOpLabels[label] = count / gpuDecodeRecordOpLabelSampleCount;
1216
+ }
1217
+ }
1010
1218
  const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
1219
+ || gpuDecodeRecordOps.length > 0 || gpuDecodeRecordPasses.length > 0
1220
+ || gpuDecodeRecordMsPerOp.length > 0 || gpuDecodeRecordMsPerPass.length > 0
1221
+ || gpuDecodeRecordPassesPerOp.length > 0
1222
+ || gpuDecodeRecordMsPerExecutedBatchToken.length > 0
1223
+ || gpuDecodeRecordOpsPerExecutedBatchToken.length > 0
1224
+ || gpuDecodeRecordPassesPerExecutedBatchToken.length > 0
1225
+ || hasGpuDecodeRecordOpLabels
1011
1226
  || gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
1227
+ || gpuDecodeReadbackMapWaitMs.length > 0 || gpuDecodeReadbackCleanupMs.length > 0
1228
+ || gpuDecodeReadbackCopyMs.length > 0
1012
1229
  || gpuDecodeOrchestrationMs.length > 0
1230
+ || lastGpuUniformCache
1013
1231
  || singleTokenSubmitWaitMs.length > 0 || singleTokenReadbackWaitMs.length > 0
1232
+ || singleTokenReadbackMapWaitMs.length > 0 || singleTokenReadbackCleanupMs.length > 0
1233
+ || singleTokenReadbackCopyMs.length > 0
1014
1234
  || singleTokenOrchestrationMs.length > 0;
1015
1235
  const gpuPhaseStats = hasGpuStats
1016
1236
  ? {
1017
1237
  prefillMs: computeSampleStats(gpuPrefillMs),
1018
1238
  decodeMs: computeSampleStats(gpuDecodeMs),
1019
1239
  decodeRecordMs: computeSampleStats(gpuDecodeRecordMs),
1240
+ decodeRecordOps: gpuDecodeRecordOpsStats,
1241
+ decodeRecordPasses: gpuDecodeRecordPassesStats,
1242
+ decodeRecordMsPerOp: computeSampleStats(gpuDecodeRecordMsPerOp),
1243
+ decodeRecordMsPerPass: computeSampleStats(gpuDecodeRecordMsPerPass),
1244
+ decodeRecordPassesPerOp: computeSampleStats(gpuDecodeRecordPassesPerOp),
1245
+ decodeRecordMsPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordMsPerExecutedBatchToken),
1246
+ decodeRecordOpsPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordOpsPerExecutedBatchToken),
1247
+ decodeRecordPassesPerExecutedBatchToken: computeSampleStats(gpuDecodeRecordPassesPerExecutedBatchToken),
1248
+ decodeRecordUniqueOpLabels: hasGpuDecodeRecordOpLabels ? Object.keys(gpuDecodeRecordOpLabels).length : null,
1249
+ decodeRecordTopOps: hasGpuDecodeRecordOpLabels
1250
+ ? buildDecodeRecordTopOps(
1251
+ gpuDecodeRecordMeanOpLabels,
1252
+ gpuDecodeRecordOpsStats?.mean
1253
+ )
1254
+ : [],
1255
+ decodeRecordTopOpGroups: hasGpuDecodeRecordOpLabels
1256
+ ? buildDecodeRecordTopOpGroups(
1257
+ gpuDecodeRecordMeanOpLabels,
1258
+ gpuDecodeRecordOpsStats?.mean
1259
+ )
1260
+ : [],
1020
1261
  decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
1021
1262
  decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
1263
+ decodeReadbackMapWaitMs: computeSampleStats(gpuDecodeReadbackMapWaitMs),
1264
+ decodeReadbackCleanupMs: computeSampleStats(gpuDecodeReadbackCleanupMs),
1265
+ decodeReadbackCopyMs: computeSampleStats(gpuDecodeReadbackCopyMs),
1022
1266
  decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
1023
1267
  prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
1024
1268
  prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
1269
+ uniformCache: lastGpuUniformCache,
1025
1270
  singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
1026
1271
  singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
1272
+ singleTokenReadbackMapWaitMs: computeSampleStats(singleTokenReadbackMapWaitMs),
1273
+ singleTokenReadbackCleanupMs: computeSampleStats(singleTokenReadbackCleanupMs),
1274
+ singleTokenReadbackCopyMs: computeSampleStats(singleTokenReadbackCopyMs),
1027
1275
  singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
1028
1276
  }
1029
1277
  : null;
@@ -1031,7 +1279,13 @@ async function runBenchSuite(options = {}) {
1031
1279
  || unbatchedForwardCalls.length > 0
1032
1280
  || totalBatchedTimeMs.length > 0
1033
1281
  || totalUnbatchedTimeMs.length > 0
1034
- || gpuSubmissions.length > 0;
1282
+ || gpuSubmissions.length > 0
1283
+ || requestedBatchTokens.length > 0
1284
+ || effectiveBatchTokens.length > 0
1285
+ || executedBatchTokens.length > 0
1286
+ || resolvedBatchTokens.length > 0
1287
+ || maxBatchTokenCap.length > 0
1288
+ || batchClampCount.length > 0;
1035
1289
  const batchingPhaseStats = hasBatchingStats
1036
1290
  ? {
1037
1291
  batchedForwardCalls: computeSampleStats(batchedForwardCalls),
@@ -1039,6 +1293,12 @@ async function runBenchSuite(options = {}) {
1039
1293
  totalBatchedTimeMs: computeSampleStats(totalBatchedTimeMs),
1040
1294
  totalUnbatchedTimeMs: computeSampleStats(totalUnbatchedTimeMs),
1041
1295
  gpuSubmissions: computeSampleStats(gpuSubmissions),
1296
+ requestedBatchTokens: computeSampleStats(requestedBatchTokens),
1297
+ effectiveBatchTokens: computeSampleStats(effectiveBatchTokens),
1298
+ executedBatchTokens: computeSampleStats(executedBatchTokens),
1299
+ resolvedBatchTokens: computeSampleStats(resolvedBatchTokens),
1300
+ maxBatchTokenCap: computeSampleStats(maxBatchTokenCap),
1301
+ batchClampCount: computeSampleStats(batchClampCount),
1042
1302
  }
1043
1303
  : null;
1044
1304
  const hasPlePreparedTokenCacheStats = plePreparedTokenCacheHits.length > 0
@@ -1069,8 +1329,8 @@ async function runBenchSuite(options = {}) {
1069
1329
  warmupRuns,
1070
1330
  timedRuns,
1071
1331
  ...(Number.isFinite(benchRun?.seed) ? { seed: benchRun.seed } : {}),
1072
- prompt: benchRun.promptLabel,
1073
- maxTokens: benchRun.maxTokens,
1332
+ prompt: lastPromptLabel,
1333
+ maxTokens: lastMaxTokens,
1074
1334
  decodeTokensPerSec: sampleTimingNumber(decodeTokensPerSecStats, 'median'),
1075
1335
  avgTokensGenerated: Math.round(tokensGeneratedStats.mean),
1076
1336
  avgPrefillTokens: Math.round(prefillTokensStats.mean),
@@ -1089,6 +1349,7 @@ async function runBenchSuite(options = {}) {
1089
1349
  decodeMsPerTokenP95: sampleTimingNumber(decodeMsPerTokenStats, 'p95'),
1090
1350
  decodeMsPerTokenP99: sampleTimingNumber(decodeMsPerTokenStats, 'p99'),
1091
1351
  avgPrefillMs: sampleTimingNumber(prefillMsStats, 'mean'),
1352
+ wallRunMs: sampleTimingNumber(wallRunMsStats, 'median'),
1092
1353
  modelLoadMs: safeModelLoadMs,
1093
1354
  throughput: {
1094
1355
  tokensPerSec: tokensPerSecStats,
@@ -1098,6 +1359,7 @@ async function runBenchSuite(options = {}) {
1098
1359
  },
1099
1360
  latency: {
1100
1361
  totalMs: totalMsStats,
1362
+ wallRunMs: wallRunMsStats,
1101
1363
  prefillMs: prefillMsStats,
1102
1364
  decodeMs: decodeMsStats,
1103
1365
  firstTokenMs: ttftMsStats,
@@ -1109,10 +1371,13 @@ async function runBenchSuite(options = {}) {
1109
1371
  },
1110
1372
  gpu: gpuPhaseStats,
1111
1373
  batching: batchingPhaseStats,
1374
+ decodeCadence: resolveDecodeCadence(getRuntimeConfig(), lastExecutionPlan),
1112
1375
  plePreparedTokenCache: plePreparedTokenCacheStats,
1113
1376
  decodeMode: lastDecodeMode,
1114
1377
  batchGuardReason: lastBatchGuardReason,
1378
+ executionPlan: lastExecutionPlan,
1115
1379
  generatedText,
1380
+ referenceTranscript: generatedReferenceTranscript,
1116
1381
  promptInput: generatedPromptInput,
1117
1382
  };
1118
1383
 
@@ -1139,6 +1404,16 @@ async function runBenchSuite(options = {}) {
1139
1404
  const memoryStats = typeof harness.pipeline?.getMemoryStats === 'function'
1140
1405
  ? harness.pipeline.getMemoryStats()
1141
1406
  : null;
1407
+ const loadTimings = resolvePipelineLoadTimings(harness.pipeline);
1408
+ const loadDiagnostics = buildLoadTimingDiagnostics(
1409
+ safeModelLoadMs,
1410
+ loadTimings.loadTiming,
1411
+ loadTimings.pipelineLoadTiming
1412
+ );
1413
+ const decodeBottleneck = buildDecodeBottleneckDiagnostics(metrics, timing);
1414
+ if (decodeBottleneck) {
1415
+ metrics.decodeBottleneck = decodeBottleneck;
1416
+ }
1142
1417
 
1143
1418
  if (typeof harness.pipeline.unload === 'function' && !options.keepPipeline) {
1144
1419
  await harness.pipeline.unload();
@@ -1148,13 +1423,22 @@ async function runBenchSuite(options = {}) {
1148
1423
  const timingDiagnostics = buildTimingDiagnostics(timing, {
1149
1424
  source: 'doppler',
1150
1425
  prefillSemantics: 'internal_prefill_phase',
1426
+ loadTiming: loadTimings.loadTiming,
1427
+ pipelineLoadTiming: loadTimings.pipelineLoadTiming,
1151
1428
  });
1429
+ if (decodeBottleneck) {
1430
+ timingDiagnostics.decodeBottleneck = decodeBottleneck;
1431
+ }
1152
1432
  const firstLoad = buildFirstLoadComposition({
1153
1433
  modelLoadMs: timing.modelLoadMs,
1154
1434
  firstTokenMs: timing.firstTokenMs,
1155
1435
  firstResponseMs: timing.firstResponseMs,
1156
1436
  });
1157
- const metricsWithContracts = buildSuiteContractMetrics('bench', metrics, harness.manifest);
1437
+ const metricsWithContracts = buildSuiteContractMetrics(
1438
+ 'bench',
1439
+ loadDiagnostics ? { ...metrics, load: loadDiagnostics } : metrics,
1440
+ harness.manifest
1441
+ );
1158
1442
  return {
1159
1443
  ...summary,
1160
1444
  modelId: options.modelId || harness.manifest?.modelId || 'unknown',
@@ -2,6 +2,7 @@
2
2
  * Decode ring buffer pool for GPU batch decode.
3
3
  *
4
4
  * Preallocates fixed GPUBuffer rings for token outputs, stop flags, and staging readback.
5
+ * Token staging buffers reserve trailing bytes for packed finiteness status.
5
6
  * Ring sizing derives from runtime batching config (batchSize × readbackInterval).
6
7
  */
7
8
 
@@ -1,6 +1,7 @@
1
1
  import { getDevice, getDeviceLimits } from '../gpu/device.js';
2
2
 
3
3
  const TOKEN_BYTES = 4;
4
+ const FINITENESS_STATUS_BYTES = 16;
4
5
 
5
6
  function clampRingSize(size) {
6
7
  if (size == null) return 0;
@@ -124,13 +125,14 @@ export class DecodeRing {
124
125
 
125
126
  const tokensBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
126
127
  const stopBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
127
- const stagingBytes = normalized.tokensPerInterval * TOKEN_BYTES;
128
+ const tokenStagingBytes = normalized.tokensPerInterval * TOKEN_BYTES + FINITENESS_STATUS_BYTES;
129
+ const stopStagingBytes = normalized.tokensPerInterval * TOKEN_BYTES;
128
130
 
129
131
  assertBufferFits('tokens', tokensBytes, true, limits);
130
- assertBufferFits('stagingTokens', stagingBytes, false, limits);
132
+ assertBufferFits('stagingTokens', tokenStagingBytes, false, limits);
131
133
  if (normalized.stopCheckMode === 'per-token') {
132
134
  assertBufferFits('stop', stopBytes, true, limits);
133
- assertBufferFits('stagingStop', stagingBytes, false, limits);
135
+ assertBufferFits('stagingStop', stopStagingBytes, false, limits);
134
136
  }
135
137
 
136
138
  const buffers = {
@@ -155,7 +157,7 @@ export class DecodeRing {
155
157
  buffers.stagingTokens = Array.from({ length: normalized.ringStaging }, (_, i) => (
156
158
  device.createBuffer({
157
159
  label: `decode_ring_staging_tokens_${i}`,
158
- size: stagingBytes,
160
+ size: tokenStagingBytes,
159
161
  usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
160
162
  })
161
163
  ));
@@ -175,7 +177,7 @@ export class DecodeRing {
175
177
  buffers.stagingStop = Array.from({ length: normalized.ringStaging }, (_, i) => (
176
178
  device.createBuffer({
177
179
  label: `decode_ring_staging_stop_${i}`,
178
- size: stagingBytes,
180
+ size: stopStagingBytes,
179
181
  usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
180
182
  })
181
183
  ));
@@ -94,6 +94,31 @@ export class KVCache {
94
94
  tokenIds?: number[] | null
95
95
  ): void | Promise<void>;
96
96
 
97
+ /**
98
+ * Record direct f32 source to f16 GPU KV cache update.
99
+ * Does NOT submit - caller is responsible for submitting the recorder.
100
+ */
101
+ recordUpdateF32ToF16FromGPU(
102
+ recorder: import('../../gpu/kernel-selector.js').CommandRecorder,
103
+ layerIdx: number,
104
+ keysBuffer: GPUBuffer,
105
+ valuesBuffer: GPUBuffer,
106
+ startPos: number,
107
+ numTokens: number,
108
+ tokenIds?: number[] | null
109
+ ): void | Promise<void>;
110
+
111
+ /**
112
+ * Record metadata for a fused projection kernel that already wrote f16 K/V
113
+ * into the contiguous GPU cache through storage-buffer side effects.
114
+ */
115
+ recordF16UpdateAlreadyWrittenFromGPU(
116
+ layerIdx: number,
117
+ startPos: number,
118
+ numTokens: number,
119
+ tokenIds?: number[] | null
120
+ ): void;
121
+
97
122
  /**
98
123
  * Get cached keys and values for a layer
99
124
  */