doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -47,8 +47,107 @@ const DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET = 70;
47
47
  const DEFAULT_HARNESS_MAX_TOKENS = 32;
48
48
  const EMBEDDING_PREVIEW_LENGTH = 16;
49
49
  const GENERATION_TOKEN_DIAGNOSTIC_LIMIT = 32;
50
+ const DECODE_RECORD_TOP_OP_LIMIT = 20;
50
51
  let defaultsWarningEmitted = false;
51
52
 
53
+ export function normalizeDecodeRecordOpLabels(value) {
54
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
55
+ return null;
56
+ }
57
+ const entries = [];
58
+ for (const [label, rawCount] of Object.entries(value)) {
59
+ const count = Number(rawCount);
60
+ if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
61
+ continue;
62
+ }
63
+ entries.push([label, count]);
64
+ }
65
+ if (entries.length === 0) {
66
+ return null;
67
+ }
68
+ entries.sort((a, b) => {
69
+ const countDelta = b[1] - a[1];
70
+ return countDelta !== 0 ? countDelta : a[0].localeCompare(b[0]);
71
+ });
72
+ return Object.fromEntries(entries);
73
+ }
74
+
75
+ export function buildDecodeRecordTopOps(labelCounts, totalOps = null, limit = DECODE_RECORD_TOP_OP_LIMIT) {
76
+ const normalized = normalizeDecodeRecordOpLabels(labelCounts);
77
+ if (!normalized) {
78
+ return [];
79
+ }
80
+ const entries = Object.entries(normalized);
81
+ const denominator = Number.isFinite(totalOps) && totalOps > 0
82
+ ? totalOps
83
+ : entries.reduce((sum, [, count]) => sum + count, 0);
84
+ const maxEntries = Number.isFinite(limit) && limit > 0
85
+ ? Math.floor(limit)
86
+ : DECODE_RECORD_TOP_OP_LIMIT;
87
+ return entries.slice(0, maxEntries).map(([label, count]) => ({
88
+ label,
89
+ count,
90
+ shareOfOps: denominator > 0 ? count / denominator : null,
91
+ }));
92
+ }
93
+
94
+ function normalizeDecodeRecordOpGroupLabel(label) {
95
+ const grouped = label
96
+ .replace(/^L\d+[.:]/, '')
97
+ .replace(/:L\d+(?=:|$)/g, '');
98
+ return grouped.length > 0 ? grouped : label;
99
+ }
100
+
101
+ export function groupDecodeRecordOpLabels(labelCounts) {
102
+ const normalized = normalizeDecodeRecordOpLabels(labelCounts);
103
+ if (!normalized) {
104
+ return null;
105
+ }
106
+ const groups = {};
107
+ for (const [label, count] of Object.entries(normalized)) {
108
+ const groupLabel = normalizeDecodeRecordOpGroupLabel(label);
109
+ groups[groupLabel] = (groups[groupLabel] ?? 0) + count;
110
+ }
111
+ return normalizeDecodeRecordOpLabels(groups);
112
+ }
113
+
114
+ export function buildDecodeRecordTopOpGroups(labelCounts, totalOps = null, limit = DECODE_RECORD_TOP_OP_LIMIT) {
115
+ return buildDecodeRecordTopOps(
116
+ groupDecodeRecordOpLabels(labelCounts),
117
+ totalOps,
118
+ limit
119
+ );
120
+ }
121
+
122
+ function nonNegativeFiniteNumber(value) {
123
+ const number = Number(value);
124
+ return Number.isFinite(number) && number >= 0 ? number : null;
125
+ }
126
+
127
+ export function normalizeUniformCacheStats(value) {
128
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
129
+ return null;
130
+ }
131
+ const hits = nonNegativeFiniteNumber(value.hits);
132
+ const misses = nonNegativeFiniteNumber(value.misses);
133
+ const evictions = nonNegativeFiniteNumber(value.evictions);
134
+ const currentSize = nonNegativeFiniteNumber(value.currentSize);
135
+ const pendingDestruction = nonNegativeFiniteNumber(value.pendingDestruction);
136
+ const totalLookups = Number.isFinite(hits) && Number.isFinite(misses)
137
+ ? hits + misses
138
+ : null;
139
+ const stats = {};
140
+ if (Number.isFinite(hits)) stats.hits = hits;
141
+ if (Number.isFinite(misses)) stats.misses = misses;
142
+ if (Number.isFinite(totalLookups)) stats.totalLookups = totalLookups;
143
+ if (Number.isFinite(totalLookups) && totalLookups > 0) stats.hitRateRatio = hits / totalLookups;
144
+ if (typeof value.hitRate === 'string' && value.hitRate.length > 0) stats.hitRate = value.hitRate;
145
+ if (Number.isFinite(evictions)) stats.evictions = evictions;
146
+ if (Number.isFinite(currentSize)) stats.currentSize = currentSize;
147
+ if (Number.isFinite(pendingDestruction)) stats.pendingDestruction = pendingDestruction;
148
+ return Object.keys(stats).length > 0 ? stats : null;
149
+ }
150
+
52
151
  function normalizeTokenIdArray(value, label) {
53
152
  const raw = ArrayBuffer.isView(value) ? Array.from(value) : value;
54
153
  if (!Array.isArray(raw)) {
@@ -103,6 +202,35 @@ function digestBytes(bytes) {
103
202
  return `sha256:${sha256BytesHex(bytes)}`;
104
203
  }
105
204
 
205
+ function selectTopLogits(logits, limit, decodeToken) {
206
+ const top = [];
207
+ const count = Math.max(1, Math.floor(limit));
208
+ for (let tokenId = 0; tokenId < logits.length; tokenId++) {
209
+ const logit = logits[tokenId];
210
+ if (!Number.isFinite(logit)) continue;
211
+ const candidate = { tokenId, logit };
212
+ let insertAt = top.length;
213
+ while (
214
+ insertAt > 0
215
+ && (
216
+ candidate.logit > top[insertAt - 1].logit
217
+ || (candidate.logit === top[insertAt - 1].logit && candidate.tokenId < top[insertAt - 1].tokenId)
218
+ )
219
+ ) {
220
+ insertAt -= 1;
221
+ }
222
+ top.splice(insertAt, 0, candidate);
223
+ if (top.length > count) {
224
+ top.pop();
225
+ }
226
+ }
227
+ return top.map((entry) => ({
228
+ tokenId: entry.tokenId,
229
+ logit: entry.logit,
230
+ text: typeof decodeToken === 'function' ? decodeToken(entry.tokenId) : null,
231
+ }));
232
+ }
233
+
106
234
  function getReferenceTranscriptRuntimeConfig(runtimeConfig) {
107
235
  const config = runtimeConfig?.shared?.harness?.referenceTranscript;
108
236
  return isPlainObject(config) ? config : null;
@@ -135,6 +263,8 @@ export function digestLogitsForTranscript(logits, context) {
135
263
  throw new Error('reference transcript logits capture requires Float32Array logits.');
136
264
  }
137
265
  const digest = digestBytes(bytesFromArrayBufferView(logits));
266
+ const topK = Number.isInteger(context?.topK) ? Math.max(1, context.topK) : 8;
267
+ const decodeToken = typeof context?.decodeToken === 'function' ? context.decodeToken : null;
138
268
  return {
139
269
  index: Number.isInteger(context?.index) ? context.index : null,
140
270
  tokenId: Number.isInteger(context?.tokenId) ? context.tokenId : null,
@@ -142,6 +272,7 @@ export function digestLogitsForTranscript(logits, context) {
142
272
  dtype: 'f32',
143
273
  elementCount: logits.length,
144
274
  digest,
275
+ top: selectTopLogits(logits, topK, decodeToken),
145
276
  };
146
277
  }
147
278
 
@@ -224,15 +355,24 @@ export async function captureKvCacheByteProof(pipeline, enabled) {
224
355
  };
225
356
  }
226
357
 
227
- function warnIfUsingDefaults(runtimeConfig) {
358
+ function hasPromptInput(value) {
359
+ return (typeof value === 'string' && value.trim().length > 0)
360
+ || (Array.isArray(value) && value.length > 0)
361
+ || isStructuredPromptInput(value);
362
+ }
363
+
364
+ function warnIfUsingDefaults(runtimeConfig, runOverrides = null) {
228
365
  if (defaultsWarningEmitted) return;
229
366
  const promptOverride = runtimeConfig?.inference?.prompt;
230
- const hasPrompt = (typeof promptOverride === 'string' && promptOverride.trim().length > 0)
231
- || (Array.isArray(promptOverride) && promptOverride.length > 0)
232
- || isStructuredPromptInput(promptOverride);
233
- const hasSampling = isPlainObject(runtimeConfig?.inference?.sampling)
367
+ const hasPrompt = hasPromptInput(runOverrides?.prompt)
368
+ || hasPromptInput(promptOverride);
369
+ const hasRuntimeSampling = isPlainObject(runtimeConfig?.inference?.sampling)
234
370
  && Object.keys(runtimeConfig.inference.sampling).length > 0;
235
- const hasMaxTokens = Number.isFinite(runtimeConfig?.inference?.generation?.maxTokens);
371
+ const hasRunSampling = isPlainObject(runOverrides?.sampling)
372
+ && Object.keys(runOverrides.sampling).length > 0;
373
+ const hasSampling = hasRuntimeSampling || hasRunSampling;
374
+ const hasMaxTokens = Number.isFinite(runOverrides?.maxTokens)
375
+ || Number.isFinite(runtimeConfig?.inference?.generation?.maxTokens);
236
376
  if (hasPrompt && hasSampling && hasMaxTokens) return;
237
377
  defaultsWarningEmitted = true;
238
378
  const missingFields = [];
@@ -1210,8 +1350,68 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1210
1350
  if (Number.isFinite(stats.gpuTimePrefillMs)) gpu.prefillMs = stats.gpuTimePrefillMs;
1211
1351
  if (Number.isFinite(stats.gpuTimeDecodeMs)) gpu.decodeMs = stats.gpuTimeDecodeMs;
1212
1352
  if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
1353
+ if (Number.isFinite(stats.decodeRecordOps)) gpu.decodeRecordOps = stats.decodeRecordOps;
1354
+ if (Number.isFinite(stats.decodeRecordPasses)) gpu.decodeRecordPasses = stats.decodeRecordPasses;
1355
+ const decodeRecordOpLabels = normalizeDecodeRecordOpLabels(stats.decodeRecordOpLabels);
1356
+ if (decodeRecordOpLabels) {
1357
+ const decodeRecordTopOps = buildDecodeRecordTopOps(decodeRecordOpLabels, stats.decodeRecordOps);
1358
+ const decodeRecordTopOpGroups = buildDecodeRecordTopOpGroups(decodeRecordOpLabels, stats.decodeRecordOps);
1359
+ gpu.decodeRecordOpLabels = decodeRecordOpLabels;
1360
+ gpu.decodeRecordUniqueOpLabels = Object.keys(decodeRecordOpLabels).length;
1361
+ gpu.decodeRecordTopOps = decodeRecordTopOps;
1362
+ gpu.decodeRecordTopOpGroups = decodeRecordTopOpGroups;
1363
+ }
1364
+ if (
1365
+ Number.isFinite(stats.decodeRecordMs) &&
1366
+ Number.isFinite(stats.decodeRecordOps) &&
1367
+ stats.decodeRecordOps > 0
1368
+ ) {
1369
+ gpu.decodeRecordMsPerOp = stats.decodeRecordMs / stats.decodeRecordOps;
1370
+ }
1371
+ if (
1372
+ Number.isFinite(stats.decodeRecordMs) &&
1373
+ Number.isFinite(stats.decodeRecordPasses) &&
1374
+ stats.decodeRecordPasses > 0
1375
+ ) {
1376
+ gpu.decodeRecordMsPerPass = stats.decodeRecordMs / stats.decodeRecordPasses;
1377
+ }
1378
+ if (
1379
+ Number.isFinite(stats.decodeRecordOps) &&
1380
+ Number.isFinite(stats.decodeRecordPasses) &&
1381
+ stats.decodeRecordOps > 0
1382
+ ) {
1383
+ gpu.decodeRecordPassesPerOp = stats.decodeRecordPasses / stats.decodeRecordOps;
1384
+ }
1385
+ if (
1386
+ Number.isFinite(stats.decodeRecordPasses) &&
1387
+ Number.isFinite(stats.batching?.executedBatchTokens) &&
1388
+ stats.batching.executedBatchTokens > 0
1389
+ ) {
1390
+ gpu.decodeRecordPassesPerExecutedBatchToken = stats.decodeRecordPasses / stats.batching.executedBatchTokens;
1391
+ }
1392
+ if (
1393
+ Number.isFinite(stats.decodeRecordMs) &&
1394
+ Number.isFinite(stats.batching?.executedBatchTokens) &&
1395
+ stats.batching.executedBatchTokens > 0
1396
+ ) {
1397
+ gpu.decodeRecordMsPerExecutedBatchToken = stats.decodeRecordMs / stats.batching.executedBatchTokens;
1398
+ }
1399
+ if (
1400
+ Number.isFinite(stats.decodeRecordOps) &&
1401
+ Number.isFinite(stats.batching?.executedBatchTokens) &&
1402
+ stats.batching.executedBatchTokens > 0
1403
+ ) {
1404
+ gpu.decodeRecordOpsPerExecutedBatchToken = stats.decodeRecordOps / stats.batching.executedBatchTokens;
1405
+ }
1406
+ const uniformCacheStats = normalizeUniformCacheStats(stats.uniformCache);
1407
+ if (uniformCacheStats) {
1408
+ gpu.uniformCache = uniformCacheStats;
1409
+ }
1213
1410
  if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
1214
1411
  if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
1412
+ if (Number.isFinite(stats.decodeReadbackMapWaitMs)) gpu.decodeReadbackMapWaitMs = stats.decodeReadbackMapWaitMs;
1413
+ if (Number.isFinite(stats.decodeReadbackCleanupMs)) gpu.decodeReadbackCleanupMs = stats.decodeReadbackCleanupMs;
1414
+ if (Number.isFinite(stats.decodeReadbackCopyMs)) gpu.decodeReadbackCopyMs = stats.decodeReadbackCopyMs;
1215
1415
  if (Number.isFinite(stats.prefillRecordMs)) gpu.prefillRecordMs = stats.prefillRecordMs;
1216
1416
  if (Number.isFinite(stats.prefillSubmitWaitMs)) gpu.prefillSubmitWaitMs = stats.prefillSubmitWaitMs;
1217
1417
  if (
@@ -1220,10 +1420,14 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1220
1420
  Number.isFinite(stats.decodeSubmitWaitMs) &&
1221
1421
  Number.isFinite(stats.decodeReadbackWaitMs)
1222
1422
  ) {
1223
- gpu.decodeOrchestrationMs = decodeMs - stats.decodeRecordMs - stats.decodeSubmitWaitMs - stats.decodeReadbackWaitMs;
1423
+ const decodeGpuWaitMs = Math.max(stats.decodeSubmitWaitMs, stats.decodeReadbackWaitMs);
1424
+ gpu.decodeOrchestrationMs = decodeMs - stats.decodeRecordMs - decodeGpuWaitMs;
1224
1425
  }
1225
1426
  if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
1226
1427
  if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
1428
+ if (Number.isFinite(stats.singleTokenReadbackMapWaitMs)) gpu.singleTokenReadbackMapWaitMs = stats.singleTokenReadbackMapWaitMs;
1429
+ if (Number.isFinite(stats.singleTokenReadbackCleanupMs)) gpu.singleTokenReadbackCleanupMs = stats.singleTokenReadbackCleanupMs;
1430
+ if (Number.isFinite(stats.singleTokenReadbackCopyMs)) gpu.singleTokenReadbackCopyMs = stats.singleTokenReadbackCopyMs;
1227
1431
  if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
1228
1432
  const gpuPhase = Object.keys(gpu).length > 0 ? gpu : null;
1229
1433
  const batching = {};
@@ -1242,6 +1446,24 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1242
1446
  if (Number.isFinite(stats.batching?.gpuSubmissions)) {
1243
1447
  batching.gpuSubmissions = stats.batching.gpuSubmissions;
1244
1448
  }
1449
+ if (Number.isFinite(stats.batching?.requestedBatchTokens)) {
1450
+ batching.requestedBatchTokens = stats.batching.requestedBatchTokens;
1451
+ }
1452
+ if (Number.isFinite(stats.batching?.effectiveBatchTokens)) {
1453
+ batching.effectiveBatchTokens = stats.batching.effectiveBatchTokens;
1454
+ }
1455
+ if (Number.isFinite(stats.batching?.executedBatchTokens)) {
1456
+ batching.executedBatchTokens = stats.batching.executedBatchTokens;
1457
+ }
1458
+ if (Number.isFinite(stats.batching?.resolvedBatchTokens)) {
1459
+ batching.resolvedBatchTokens = stats.batching.resolvedBatchTokens;
1460
+ }
1461
+ if (Number.isFinite(stats.batching?.maxBatchTokenCap)) {
1462
+ batching.maxBatchTokenCap = stats.batching.maxBatchTokenCap;
1463
+ }
1464
+ if (Number.isFinite(stats.batching?.batchClampCount)) {
1465
+ batching.batchClampCount = stats.batching.batchClampCount;
1466
+ }
1245
1467
  const batchingPhase = Object.keys(batching).length > 0 ? batching : null;
1246
1468
  const plePreparedTokenCache = {};
1247
1469
  if (Number.isFinite(stats.plePreparedTokenCacheHits)) {
@@ -1265,10 +1487,12 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1265
1487
  const plePreparedTokenCachePhase = Object.keys(plePreparedTokenCache).length > 0
1266
1488
  ? plePreparedTokenCache
1267
1489
  : null;
1490
+ const wallMs = Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : durationMs;
1268
1491
 
1269
1492
  return {
1270
1493
  phase: {
1271
- totalMs: Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : durationMs,
1494
+ totalMs: prefillMs + decodeMs,
1495
+ wallMs,
1272
1496
  ttftMs,
1273
1497
  prefillMs,
1274
1498
  decodeMs,
@@ -1321,7 +1545,7 @@ export function isCoherentOutput(tokens, output) {
1321
1545
  }
1322
1546
 
1323
1547
  export async function runGeneration(pipeline, runtimeConfig, runOverrides = null) {
1324
- warnIfUsingDefaults(runtimeConfig);
1548
+ warnIfUsingDefaults(runtimeConfig, runOverrides);
1325
1549
  const tokens = [];
1326
1550
  const tokenIds = [];
1327
1551
  const tokenRecords = [];
@@ -1347,7 +1571,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
1347
1571
  const profile = runtimeConfig.shared?.debug?.profiler?.enabled === true;
1348
1572
  const explicitDiagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
1349
1573
  || shouldEnableReferenceTranscriptDiagnostics(runOverrides, runtimeConfig);
1350
- const disableCommandBatching = explicitDiagnosticsEnabled
1574
+ const disableCommandBatchingForDiagnostics = explicitDiagnosticsEnabled
1351
1575
  || (Array.isArray(debugProbes) && debugProbes.length > 0);
1352
1576
  const start = performance.now();
1353
1577
  const diagnostics = resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides);
@@ -1363,13 +1587,14 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
1363
1587
  greedyThreshold: sampling.greedyThreshold,
1364
1588
  useChatTemplate,
1365
1589
  profile,
1366
- disableCommandBatching,
1590
+ ...(disableCommandBatchingForDiagnostics ? { disableCommandBatching: true } : {}),
1367
1591
  diagnostics,
1368
1592
  ...(captureLogits ? {
1369
1593
  onLogits: (logits, context) => {
1370
1594
  logitsDigests.push(digestLogitsForTranscript(logits, {
1371
1595
  ...context,
1372
1596
  index: logitsDigests.length,
1597
+ decodeToken: (tokenId) => pipeline?.tokenizer?.decode?.([tokenId], false, false) ?? null,
1373
1598
  }));
1374
1599
  },
1375
1600
  } : {}),
@@ -192,6 +192,14 @@ export declare function applyRuntimeForRun(
192
192
  options?: RuntimeConfigLoadOptions & { runtime?: RuntimeOverrides }
193
193
  ): Promise<void>;
194
194
 
195
+ export declare function resolveExecutionGraphHash(
196
+ manifest: Record<string, unknown> | null | undefined
197
+ ): string | null;
198
+
199
+ export declare function buildReferenceTranscriptSeed(
200
+ run: Record<string, unknown>,
201
+ context?: Record<string, unknown>
202
+ ): Record<string, unknown>;
195
203
 
196
204
  export declare function runBrowserSuite(
197
205
  options: BrowserSuiteOptions