doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -11,6 +11,7 @@ import {
11
11
  getActiveKernelPathPolicy,
12
12
  } from '../config/kernel-path-loader.js';
13
13
  import { validateTrainingMetricsReport } from '../config/schema/training-metrics.schema.js';
14
+ import { modelSupportsEmbedding } from '../config/schema/manifest.schema.js';
14
15
  import {
15
16
  resolveReportTimestamp,
16
17
  resolveRuntime,
@@ -33,6 +34,7 @@ import {
33
34
  normalizeWorkloadType,
34
35
  assertDiffusionPerformanceArtifact,
35
36
  toTimingNumber,
37
+ buildFirstLoadComposition,
36
38
  safeToFixed,
37
39
  sampleTimingNumber,
38
40
  buildCanonicalTiming,
@@ -56,6 +58,8 @@ import {
56
58
  runEnergySuite,
57
59
  } from './browser-harness-diffusion-energy-suites.js';
58
60
  import { collectTrainingArtifactsFromSuiteResult } from './browser-harness-report-helpers.js';
61
+ import { sha256Hex } from '../utils/sha256.js';
62
+ import { stableSortObject } from '../utils/stable-sort-object.js';
59
63
 
60
64
  const TRAINING_SUITE_MODULE_PATH = '../experimental/training/suite.js';
61
65
  let trainingSuiteModulePromise = null;
@@ -67,15 +71,6 @@ async function loadTrainingSuiteModule() {
67
71
  return trainingSuiteModulePromise;
68
72
  }
69
73
 
70
- /**
71
- * Clear the cached training suite module promise.
72
- * Useful for testing or when the training module needs to be re-imported
73
- * after configuration changes.
74
- */
75
- export function clearTrainingSuiteModule() {
76
- trainingSuiteModulePromise = null;
77
- }
78
-
79
74
  export async function runTrainingSuite(options = {}) {
80
75
  const module = await loadTrainingSuiteModule();
81
76
  return module.runTrainingSuite(options);
@@ -95,48 +90,6 @@ async function runTrainingBenchSuite(options = {}) {
95
90
  return module.runTrainingBenchSuite(options);
96
91
  }
97
92
 
98
- export async function initializeBrowserHarness(options = {}) {
99
- const { modelUrl, onProgress, log } = options;
100
- if (!modelUrl) {
101
- throw new Error('modelUrl is required');
102
- }
103
-
104
- const runtime = resolveRuntime(options);
105
- const result = await initializeInference(modelUrl, {
106
- runtime,
107
- onProgress,
108
- log,
109
- });
110
-
111
- return { ...result, runtime };
112
- }
113
-
114
- export async function saveBrowserReport(modelId, report, options = {}) {
115
- return saveReport(modelId, report, options);
116
- }
117
-
118
- export async function runBrowserHarness(options = {}) {
119
- const harness = await initializeBrowserHarness(options);
120
- const reportTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserHarness timestamp');
121
- const modelId = options.modelId || harness.manifest?.modelId || 'unknown';
122
-
123
- let report = options.report || null;
124
- if (!report && typeof options.buildReport === 'function') {
125
- report = await options.buildReport(harness);
126
- }
127
- if (!report) {
128
- report = {
129
- modelId,
130
- timestamp: reportTimestamp,
131
- };
132
- } else if (!report.timestamp) {
133
- report = { ...report, timestamp: reportTimestamp };
134
- }
135
-
136
- const reportInfo = await saveReport(modelId, report, { timestamp: report.timestamp || reportTimestamp });
137
- return { ...harness, report, reportInfo };
138
- }
139
-
140
93
  const BROWSER_WORKLOAD_SET = Object.freeze([
141
94
  'kernels',
142
95
  'inference',
@@ -296,6 +249,147 @@ function resolveDispatchSuite(mode, workload) {
296
249
  return workload;
297
250
  }
298
251
 
252
+ function stableJson(value) {
253
+ return JSON.stringify(stableSortObject(value)) ?? 'null';
254
+ }
255
+
256
+ function hashStableJson(value) {
257
+ return `sha256:${sha256Hex(stableJson(value))}`;
258
+ }
259
+
260
+ export function resolveExecutionGraphHash(manifest) {
261
+ const execution = manifest?.inference?.execution;
262
+ if (!execution || typeof execution !== 'object') {
263
+ return null;
264
+ }
265
+ return hashStableJson(execution);
266
+ }
267
+
268
+ function buildPerStepTokenProof(tokenIds) {
269
+ return tokenIds.map((tokenId, index) => ({
270
+ index,
271
+ tokenId,
272
+ tokenHash: hashStableJson({ index, tokenId }),
273
+ }));
274
+ }
275
+
276
+ function buildKvCacheTranscriptSeed(kvCache, byteProof = null) {
277
+ const source = kvCache && typeof kvCache === 'object' ? kvCache : null;
278
+ const proof = byteProof && typeof byteProof === 'object' ? byteProof : null;
279
+ const seed = {
280
+ mode: proof ? 'stats+sha256-layer-kv-bytes' : (source ? 'stats' : 'not-captured'),
281
+ layout: typeof source?.layout === 'string' ? source.layout : null,
282
+ kvDtype: typeof source?.kvDtype === 'string' ? source.kvDtype : null,
283
+ seqLen: Number.isFinite(source?.seqLen) ? source.seqLen : null,
284
+ maxSeqLen: Number.isFinite(source?.maxSeqLen) ? source.maxSeqLen : null,
285
+ usedBytes: Number.isFinite(source?.usedBytes) ? source.usedBytes : null,
286
+ allocatedBytes: Number.isFinite(source?.allocatedBytes) ? source.allocatedBytes : null,
287
+ counters: source?.counters ?? null,
288
+ byteDigestMode: typeof proof?.mode === 'string' ? proof.mode : null,
289
+ byteDigest: typeof proof?.digest === 'string' ? proof.digest : null,
290
+ byteDigests: Array.isArray(proof?.layers) ? proof.layers : null,
291
+ };
292
+ return {
293
+ ...seed,
294
+ stateHash: hashStableJson(seed),
295
+ };
296
+ }
297
+
298
+ export function buildReferenceTranscriptSeed(run, context = {}) {
299
+ const promptPayload = run.promptInput ?? run.prompt ?? null;
300
+ const outputText = typeof run.output === 'string' ? run.output : '';
301
+ const tokenIds = Array.isArray(run.tokenIds)
302
+ ? run.tokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
303
+ : [];
304
+ const promptTokenIds = Array.isArray(run.promptTokenIds)
305
+ ? run.promptTokenIds.map((value) => Number(value)).filter((value) => Number.isInteger(value))
306
+ : null;
307
+ const logitsDigests = Array.isArray(run.logitsDigests)
308
+ ? run.logitsDigests
309
+ : [];
310
+ const hasCompleteLogitsDigests = logitsDigests.length === tokenIds.length && tokenIds.length > 0;
311
+ const transcript = {
312
+ schema: 'doppler.reference-transcript/v1',
313
+ source: {
314
+ kind: 'inline-browser-suite',
315
+ path: 'inline',
316
+ hash: 'sha256:' + '0'.repeat(64),
317
+ },
318
+ executionGraphHash: context.executionGraphHash ?? null,
319
+ surface: 'browser-webgpu',
320
+ prompt: {
321
+ identity: typeof run.prompt === 'string' && run.prompt.trim() ? run.prompt : 'promptInput',
322
+ hash: hashStableJson(promptPayload),
323
+ tokenIdsHash: promptTokenIds ? hashStableJson(promptTokenIds) : null,
324
+ tokenCount: promptTokenIds ? promptTokenIds.length : null,
325
+ },
326
+ output: {
327
+ textHash: `sha256:${sha256Hex(outputText)}`,
328
+ tokensGenerated: tokenIds.length,
329
+ stopReason: typeof run.phase?.stopReason === 'string' ? run.phase.stopReason : 'unknown',
330
+ stopTokenId: Number.isInteger(run.phase?.stopTokenId) ? run.phase.stopTokenId : null,
331
+ },
332
+ tokens: {
333
+ ids: tokenIds,
334
+ generatedTokenIdsHash: hashStableJson(tokenIds),
335
+ generatedTextHash: `sha256:${sha256Hex(outputText)}`,
336
+ preview: Array.isArray(run.tokenDiagnostics?.preview) ? run.tokenDiagnostics.preview : [],
337
+ perStep: buildPerStepTokenProof(tokenIds),
338
+ coverage: {
339
+ mode: 'full-token-ids',
340
+ omitted: 0,
341
+ },
342
+ },
343
+ phase: {
344
+ prefillMs: Number.isFinite(run.phase?.prefillMs) ? run.phase.prefillMs : null,
345
+ decodeMs: Number.isFinite(run.phase?.decodeMs) ? run.phase.decodeMs : null,
346
+ prefillTokens: Number.isFinite(run.phase?.prefillTokens) ? run.phase.prefillTokens : null,
347
+ decodeTokens: Number.isFinite(run.phase?.decodeTokens) ? run.phase.decodeTokens : null,
348
+ },
349
+ kvCache: buildKvCacheTranscriptSeed(
350
+ run.phase?.kvCache ?? context.kvCache ?? null,
351
+ run.kvCacheByteProof ?? null
352
+ ),
353
+ logits: hasCompleteLogitsDigests ? {
354
+ mode: 'sha256-per-step',
355
+ perStepDigests: logitsDigests.map((entry) => entry.digest),
356
+ steps: logitsDigests,
357
+ } : {
358
+ mode: 'not-captured',
359
+ reason: logitsDigests.length > 0
360
+ ? 'Per-step logits digest count did not match generated token count.'
361
+ : 'Per-step logits digests were not requested for this browser harness run.',
362
+ perStepDigests: null,
363
+ },
364
+ tolerance: {
365
+ tokenPolicy: 'exact generated token IDs',
366
+ logitsPolicy: hasCompleteLogitsDigests
367
+ ? 'exact sha256 digest per generated step over finalized f32 logits before sampling'
368
+ : 'not captured',
369
+ kvPolicy: run.kvCacheByteProof
370
+ ? 'exact sha256 digest over used KV cache bytes by layer/key/value'
371
+ : 'metadata hash only; KV tensor bytes are not read back by default',
372
+ },
373
+ };
374
+ return {
375
+ ...transcript,
376
+ source: {
377
+ ...transcript.source,
378
+ hash: hashStableJson({
379
+ prompt: transcript.prompt,
380
+ output: transcript.output,
381
+ tokens: {
382
+ generatedTokenIdsHash: transcript.tokens.generatedTokenIdsHash,
383
+ generatedTextHash: transcript.tokens.generatedTextHash,
384
+ },
385
+ phase: transcript.phase,
386
+ kvCache: transcript.kvCache,
387
+ logits: transcript.logits,
388
+ }),
389
+ },
390
+ };
391
+ }
392
+
299
393
  async function runKernelSuite(options = {}) {
300
394
  const startTime = performance.now();
301
395
  const { testHarness, initGPU } = await import('../../tests/kernels/browser/test-page.js');
@@ -324,23 +418,33 @@ async function runKernelSuite(options = {}) {
324
418
 
325
419
  async function runInferenceSuite(options = {}) {
326
420
  const startTime = performance.now();
327
- const harness = await initializeSuiteModel(options);
421
+ const cacheMode = normalizeCacheMode(options.cacheMode);
422
+ const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
423
+ const harness = await withHarnessPhase(
424
+ 'inference.initializeSuiteModel',
425
+ {
426
+ modelId: options.modelId ?? null,
427
+ loadMode,
428
+ cacheMode,
429
+ },
430
+ () => initializeSuiteModel(options)
431
+ );
328
432
  const runtimeConfig = getRuntimeConfig();
329
433
  const modelType = harness.manifest?.modelType || 'transformer';
330
- if (options.expectedModelType === 'embedding' && modelType !== 'embedding') {
434
+ const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
435
+ if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
331
436
  throw new Error(
332
- `Expected an embedding model for workload "${options.workload || 'inference'}", got "${modelType}".`
437
+ `Expected an embedding-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
438
+ `Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
333
439
  );
334
440
  }
335
- const cacheMode = normalizeCacheMode(options.cacheMode);
336
- const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
337
441
  const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
338
442
 
339
443
  let results;
340
444
  let output = null;
341
445
  let metrics;
342
446
 
343
- if (modelType === 'embedding') {
447
+ if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
344
448
  const run = await runEmbedding(harness.pipeline, runtimeConfig);
345
449
  const semantic = await runEmbeddingSemanticChecks(harness.pipeline, options);
346
450
  const isValidEmbedding = run.embeddingDim > 0 && run.nonFiniteCount === 0;
@@ -464,6 +568,8 @@ async function runInferenceSuite(options = {}) {
464
568
  decodeMs: safeToFixed(run.phase.decodeMs),
465
569
  prefillTokens: Math.round(run.phase.prefillTokens),
466
570
  decodeTokens: Math.round(run.phase.decodeTokens),
571
+ stopReason: run.phase.stopReason ?? null,
572
+ stopTokenId: Number.isInteger(run.phase.stopTokenId) ? run.phase.stopTokenId : null,
467
573
  prefillTokensPerSec: safeToFixed(run.phase.prefillTokensPerSec),
468
574
  prefillTokensPerSecTtft: safeToFixed(run.phase.prefillTokensPerSecTtft),
469
575
  decodeTokensPerSec: safeToFixed(run.phase.decodeTokensPerSec),
@@ -476,6 +582,11 @@ async function runInferenceSuite(options = {}) {
476
582
  kernelPathId: run.phase.kernelPathId,
477
583
  kernelPathSource: run.phase.kernelPathSource,
478
584
  generationDiagnostics: run.tokenDiagnostics,
585
+ kvCache: run.phase.kvCache ?? null,
586
+ referenceTranscript: buildReferenceTranscriptSeed(run, {
587
+ executionGraphHash: resolveExecutionGraphHash(harness.manifest),
588
+ kvCache: run.phase.kvCache ?? null,
589
+ }),
479
590
  operatorDiagnostics: run.phase.operatorDiagnostics ?? null,
480
591
  };
481
592
  }
@@ -509,6 +620,11 @@ async function runInferenceSuite(options = {}) {
509
620
  source: 'doppler',
510
621
  prefillSemantics: 'internal_prefill_phase',
511
622
  });
623
+ const firstLoad = buildFirstLoadComposition({
624
+ modelLoadMs: timing.modelLoadMs,
625
+ firstTokenMs: timing.firstTokenMs,
626
+ firstResponseMs: timing.firstResponseMs,
627
+ });
512
628
  const metricsWithContracts = buildSuiteContractMetrics(
513
629
  options.suiteName || 'inference',
514
630
  metrics,
@@ -530,6 +646,7 @@ async function runInferenceSuite(options = {}) {
530
646
  },
531
647
  timing,
532
648
  timingDiagnostics,
649
+ firstLoad,
533
650
  output,
534
651
  metrics: metricsWithContracts,
535
652
  memoryStats,
@@ -584,6 +701,11 @@ async function runBenchSuite(options = {}) {
584
701
  source: 'doppler',
585
702
  prefillSemantics: 'not_applicable_training_workload',
586
703
  });
704
+ const firstLoad = buildFirstLoadComposition({
705
+ modelLoadMs: timing.modelLoadMs,
706
+ firstTokenMs: timing.firstTokenMs,
707
+ firstResponseMs: timing.firstResponseMs,
708
+ });
587
709
  return {
588
710
  ...trainingBench,
589
711
  modelId: trainingBench.modelId || options.modelId || options.modelUrl || 'training',
@@ -600,6 +722,7 @@ async function runBenchSuite(options = {}) {
600
722
  },
601
723
  timing,
602
724
  timingDiagnostics,
725
+ firstLoad,
603
726
  output: null,
604
727
  memoryStats: null,
605
728
  deviceInfo: trainingBench.deviceInfo ?? resolveDeviceInfo(),
@@ -639,12 +762,22 @@ async function runBenchSuite(options = {}) {
639
762
  };
640
763
  }
641
764
 
642
- const harness = await initializeSuiteModel(options);
765
+ const harness = await withHarnessPhase(
766
+ 'bench.initializeSuiteModel',
767
+ {
768
+ modelId: options.modelId ?? null,
769
+ loadMode,
770
+ cacheMode,
771
+ },
772
+ () => initializeSuiteModel(options)
773
+ );
643
774
  const benchRun = resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
644
775
  const modelType = harness.manifest?.modelType || 'transformer';
645
- if (options.expectedModelType === 'embedding' && modelType !== 'embedding') {
776
+ const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
777
+ if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
646
778
  throw new Error(
647
- `Expected an embedding model for bench workload "${options.workload || 'inference'}", got "${modelType}".`
779
+ `Expected an embedding-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
780
+ `Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
648
781
  );
649
782
  }
650
783
  const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
@@ -654,7 +787,7 @@ async function runBenchSuite(options = {}) {
654
787
  let output = null;
655
788
  let timing;
656
789
 
657
- if (modelType === 'embedding') {
790
+ if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
658
791
  const durations = [];
659
792
  const timedDurations = [];
660
793
  const embeddingDims = [];
@@ -776,6 +909,8 @@ async function runBenchSuite(options = {}) {
776
909
  const gpuDecodeSubmitWaitMs = [];
777
910
  const gpuDecodeReadbackWaitMs = [];
778
911
  const gpuDecodeOrchestrationMs = [];
912
+ const gpuPrefillRecordMs = [];
913
+ const gpuPrefillSubmitWaitMs = [];
779
914
  const singleTokenSubmitWaitMs = [];
780
915
  const singleTokenReadbackWaitMs = [];
781
916
  const singleTokenOrchestrationMs = [];
@@ -795,10 +930,20 @@ async function runBenchSuite(options = {}) {
795
930
  let lastBatchGuardReason = null;
796
931
  for (let i = 0; i < warmupRuns + timedRuns; i++) {
797
932
  harness.pipeline.reset?.();
798
- const run = await runTextInference(harness.pipeline, runtimeConfig, {
799
- ...benchRun,
800
- ...(options.inferenceInput ?? {}),
801
- });
933
+ const run = await withHarnessPhase(
934
+ `bench.runTextInference[${i}]`,
935
+ {
936
+ modelId: options.modelId ?? harness.manifest?.modelId ?? null,
937
+ loadMode,
938
+ cacheMode,
939
+ warmupRuns,
940
+ timedRuns,
941
+ },
942
+ () => runTextInference(harness.pipeline, runtimeConfig, {
943
+ ...benchRun,
944
+ ...(options.inferenceInput ?? {}),
945
+ })
946
+ );
802
947
  if (i === warmupRuns + timedRuns - 1) {
803
948
  generatedText = run?.output ?? null;
804
949
  generatedPromptInput = run?.promptInput ?? null;
@@ -830,6 +975,8 @@ async function runBenchSuite(options = {}) {
830
975
  if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
831
976
  if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
832
977
  if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
978
+ if (Number.isFinite(phaseGpu?.prefillRecordMs)) gpuPrefillRecordMs.push(phaseGpu.prefillRecordMs);
979
+ if (Number.isFinite(phaseGpu?.prefillSubmitWaitMs)) gpuPrefillSubmitWaitMs.push(phaseGpu.prefillSubmitWaitMs);
833
980
  if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
834
981
  gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
835
982
  }
@@ -873,6 +1020,8 @@ async function runBenchSuite(options = {}) {
873
1020
  decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
874
1021
  decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
875
1022
  decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
1023
+ prefillRecordMs: computeSampleStats(gpuPrefillRecordMs),
1024
+ prefillSubmitWaitMs: computeSampleStats(gpuPrefillSubmitWaitMs),
876
1025
  singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
877
1026
  singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
878
1027
  singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
@@ -1000,6 +1149,11 @@ async function runBenchSuite(options = {}) {
1000
1149
  source: 'doppler',
1001
1150
  prefillSemantics: 'internal_prefill_phase',
1002
1151
  });
1152
+ const firstLoad = buildFirstLoadComposition({
1153
+ modelLoadMs: timing.modelLoadMs,
1154
+ firstTokenMs: timing.firstTokenMs,
1155
+ firstResponseMs: timing.firstResponseMs,
1156
+ });
1003
1157
  const metricsWithContracts = buildSuiteContractMetrics('bench', metrics, harness.manifest);
1004
1158
  return {
1005
1159
  ...summary,
@@ -1017,6 +1171,7 @@ async function runBenchSuite(options = {}) {
1017
1171
  },
1018
1172
  timing,
1019
1173
  timingDiagnostics,
1174
+ firstLoad,
1020
1175
  output,
1021
1176
  metrics: metricsWithContracts,
1022
1177
  memoryStats,
@@ -1072,6 +1227,35 @@ function shouldCaptureDebugSnapshot(mode, runtimeConfig) {
1072
1227
  || logLevel === 'verbose';
1073
1228
  }
1074
1229
 
1230
+ function createHarnessPhaseError(error, phase, context = {}) {
1231
+ const message = error?.message || String(error);
1232
+ const wrapped = new Error(
1233
+ `Browser harness phase "${phase}" failed: ${message}`,
1234
+ error instanceof Error ? { cause: error } : undefined
1235
+ );
1236
+ wrapped.name = error?.name || 'Error';
1237
+ if (error?.code !== undefined) {
1238
+ wrapped.code = error.code;
1239
+ }
1240
+ wrapped.details = {
1241
+ ...(error?.details && typeof error.details === 'object' ? error.details : {}),
1242
+ harnessPhase: phase,
1243
+ ...context,
1244
+ };
1245
+ return wrapped;
1246
+ }
1247
+
1248
+ async function withHarnessPhase(phase, context, run) {
1249
+ try {
1250
+ return await run();
1251
+ } catch (error) {
1252
+ if (error?.details?.harnessPhase) {
1253
+ throw error;
1254
+ }
1255
+ throw createHarnessPhaseError(error, phase, context);
1256
+ }
1257
+ }
1258
+
1075
1259
  export async function runBrowserSuite(options = {}) {
1076
1260
  return runWithRuntimeIsolationForSuite(async () => {
1077
1261
  const suiteTimestamp = resolveReportTimestamp(options.timestamp, 'runBrowserSuite timestamp');
@@ -1083,12 +1267,21 @@ export async function runBrowserSuite(options = {}) {
1083
1267
  if (captureDebugSnapshot) {
1084
1268
  clearLogHistory();
1085
1269
  }
1086
- const suiteResult = await dispatchBrowserSuite(mode, workload, {
1087
- ...options,
1088
- mode,
1089
- workload,
1090
- suite,
1091
- });
1270
+ const suiteResult = await withHarnessPhase(
1271
+ 'dispatchBrowserSuite',
1272
+ {
1273
+ mode,
1274
+ workload,
1275
+ modelId: options.modelId ?? null,
1276
+ loadMode: options.loadMode ?? null,
1277
+ },
1278
+ () => dispatchBrowserSuite(mode, workload, {
1279
+ ...options,
1280
+ mode,
1281
+ workload,
1282
+ suite,
1283
+ })
1284
+ );
1092
1285
  if (!suiteResult) {
1093
1286
  throw createUnsupportedWorkloadError(workload, { ...harnessContext, mode });
1094
1287
  }
@@ -42,6 +42,12 @@ export class KVCache {
42
42
  totalTokensSeen: number;
43
43
  memoryUsage: number;
44
44
  gpuContext: GPUContext | null;
45
+ counters: {
46
+ updateCalls: number;
47
+ gpuUpdateCalls: number;
48
+ recordedGpuUpdateCalls: number;
49
+ tokensWritten: number;
50
+ };
45
51
 
46
52
  /**
47
53
  * @param config - KV cache configuration
@@ -63,6 +63,13 @@ export class KVCache {
63
63
 
64
64
  this.totalTokensSeen = 0;
65
65
 
66
+ this.counters = {
67
+ updateCalls: 0,
68
+ gpuUpdateCalls: 0,
69
+ recordedGpuUpdateCalls: 0,
70
+ tokensWritten: 0,
71
+ };
72
+
66
73
  // Memory usage tracking
67
74
 
68
75
  this.memoryUsage = 0;
@@ -322,6 +329,8 @@ export class KVCache {
322
329
  if (numNewTokens === 0) {
323
330
  return;
324
331
  }
332
+ this.counters.updateCalls += 1;
333
+ this.counters.tokensWritten += numNewTokens;
325
334
 
326
335
  if (startPos + numNewTokens > this.maxSeqLen) {
327
336
  throw new Error(
@@ -362,6 +371,8 @@ export class KVCache {
362
371
  if (numTokens === 0) {
363
372
  return;
364
373
  }
374
+ this.counters.gpuUpdateCalls += 1;
375
+ this.counters.tokensWritten += numTokens;
365
376
 
366
377
  const layer = (this.layers[layerIdx]);
367
378
  const device = getDevice();
@@ -420,6 +431,8 @@ export class KVCache {
420
431
  if (numTokens === 0) {
421
432
  return;
422
433
  }
434
+ this.counters.recordedGpuUpdateCalls += 1;
435
+ this.counters.tokensWritten += numTokens;
423
436
 
424
437
  const encoder = recorder.getEncoder();
425
438
  const layer = (this.layers[layerIdx]);
@@ -820,7 +833,9 @@ export class KVCache {
820
833
  efficiency: used / actual,
821
834
  seqLen: this.currentSeqLen,
822
835
  maxSeqLen: this.maxSeqLen,
823
- layout: this.layout
836
+ layout: this.layout,
837
+ kvDtype: this.kvDtype,
838
+ counters: { ...this.counters }
824
839
  };
825
840
  }
826
841
 
@@ -3,6 +3,7 @@ import { type GPUContext, type GPUBuffersResult, type MemoryStats } from './type
3
3
  export interface MixedGeometryKVCacheConfig {
4
4
  numLayers: number;
5
5
  numHeads: number;
6
+ globalNumHeads?: number | null;
6
7
  headDim: number;
7
8
  globalHeadDim?: number | null;
8
9
  maxSeqLen: number;
@@ -25,6 +25,9 @@ function buildLayerSpecs(config, bytesPerElem) {
25
25
  ? Math.trunc(config.globalHeadDim)
26
26
  : baseHeadDim;
27
27
  const numHeads = Number(config.numHeads);
28
+ const globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
29
+ ? Math.trunc(config.globalNumHeads)
30
+ : numHeads;
28
31
  const maxSeqLen = Number(config.maxSeqLen);
29
32
  const slidingWindow = Number.isFinite(config.slidingWindow) && config.slidingWindow > 0
30
33
  ? Math.min(Math.trunc(config.slidingWindow), maxSeqLen)
@@ -46,14 +49,15 @@ function buildLayerSpecs(config, bytesPerElem) {
46
49
  const capacityTokens = isSliding && slidingWindow != null
47
50
  ? slidingWindow
48
51
  : maxSeqLen;
49
- const kvSize = numHeads * headDim;
52
+ const layerNumHeads = isSliding ? numHeads : globalNumHeads;
53
+ const kvSize = layerNumHeads * headDim;
50
54
  const bytesPerToken = kvSize * bytesPerElem;
51
55
  return {
52
56
  layerIdx,
53
57
  layerType,
54
58
  layout: isSliding && slidingWindow != null ? 'ring' : 'contiguous',
55
59
  headDim,
56
- numHeads,
60
+ numHeads: layerNumHeads,
57
61
  kvSize,
58
62
  capacityTokens,
59
63
  bytesPerToken,
@@ -120,6 +124,9 @@ export class MixedGeometryKVCache {
120
124
 
121
125
  this.numLayers = Math.trunc(config.numLayers);
122
126
  this.numHeads = Math.trunc(config.numHeads);
127
+ this.globalNumHeads = Number.isFinite(config.globalNumHeads) && config.globalNumHeads > 0
128
+ ? Math.trunc(config.globalNumHeads)
129
+ : this.numHeads;
123
130
  this.headDim = Math.max(
124
131
  Math.trunc(config.headDim),
125
132
  Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : Math.trunc(config.headDim)
@@ -142,6 +149,7 @@ export class MixedGeometryKVCache {
142
149
  ...config,
143
150
  numLayers: this.numLayers,
144
151
  numHeads: this.numHeads,
152
+ globalNumHeads: this.globalNumHeads,
145
153
  headDim: Math.trunc(config.headDim),
146
154
  globalHeadDim: Number.isFinite(config.globalHeadDim) ? Math.trunc(config.globalHeadDim) : null,
147
155
  maxSeqLen: this.maxSeqLen,
@@ -400,6 +408,8 @@ export class MixedGeometryKVCache {
400
408
  seqLen: this.currentSeqLen,
401
409
  maxSeqLen: this.maxSeqLen,
402
410
  layout: this.layout,
411
+ kvDtype: this.kvDtype,
412
+ counters: null,
403
413
  };
404
414
  }
405
415
 
@@ -350,6 +350,8 @@ export class QuantizedKVCache {
350
350
  seqLen: this.currentSeqLen,
351
351
  maxSeqLen: this.maxSeqLen,
352
352
  layout: this.layout,
353
+ kvDtype: this.kvDtype,
354
+ counters: null,
353
355
  };
354
356
  }
355
357
 
@@ -598,6 +598,8 @@ export class TieredKVCache {
598
598
  seqLen: this.currentSeqLen,
599
599
  maxSeqLen: this.maxSeqLen,
600
600
  layout: 'paged',
601
+ kvDtype: this.coldDtype,
602
+ counters: null,
601
603
  };
602
604
  return {
603
605
  theoretical: hotStats.theoretical + coldStats.theoretical,
@@ -607,6 +609,11 @@ export class TieredKVCache {
607
609
  seqLen: this.currentSeqLen,
608
610
  maxSeqLen: this.maxSeqLen,
609
611
  layout: this.layout,
612
+ kvDtype: this.kvDtype,
613
+ counters: {
614
+ hot: hotStats.counters ?? null,
615
+ cold: coldStats.counters ?? null,
616
+ },
610
617
  hot: hotStats,
611
618
  cold: coldStats,
612
619
  };
@@ -183,6 +183,8 @@ export interface MemoryStats {
183
183
  seqLen: number;
184
184
  maxSeqLen: number;
185
185
  layout: 'contiguous' | 'contiguous_quantized' | 'paged' | 'tiered' | 'bdpa_paged';
186
+ kvDtype?: 'f16' | 'f32' | null;
187
+ counters?: Record<string, unknown> | null;
186
188
  }
187
189
 
188
190
  /**
@@ -48,7 +48,7 @@ export interface ExpertTask {
48
48
  prompt: string;
49
49
  }
50
50
 
51
- export declare class MultiModelNetwork {
51
+ declare class MultiModelNetwork {
52
52
  private pipeline;
53
53
  private loader;
54
54
  private router;