doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -8,9 +8,18 @@ import {
8
8
  isWeightBuffer,
9
9
  } from '../../../gpu/weight-buffer.js';
10
10
  import { createTensor } from '../../../gpu/tensor.js';
11
- import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
11
+ import {
12
+ castF16ToF32,
13
+ castF32ToF16,
14
+ recordActivationStaticQdq,
15
+ recordCastF16ToF32,
16
+ recordCastF32ToF16,
17
+ recordScale,
18
+ runActivationStaticQdq,
19
+ runScale,
20
+ } from '../../../gpu/kernel-selector.js';
12
21
  import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
13
- import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
22
+ import { doCast, doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
14
23
  import { runProbes } from './probes.js';
15
24
  import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRangeChunkIntoOutput } from './embed.js';
16
25
  import { selectRuleValue } from '../../../rules/rule-registry.js';
@@ -19,6 +28,8 @@ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
19
28
 
20
29
  const pleRuntimeCache = new WeakMap();
21
30
  const pleRangeRowCache = new WeakMap();
31
+ const PLE_ACTIVATION_STATIC_QMIN = -128;
32
+ const PLE_ACTIVATION_STATIC_QMAX = 127;
22
33
 
23
34
  function getPerLayerInputWeights(context) {
24
35
  const weights = context.weights.get('per_layer_inputs');
@@ -63,6 +74,106 @@ function isDensePleProjectionDtype(dtype) {
63
74
  return dtype === 'f16' || dtype === 'f32';
64
75
  }
65
76
 
77
+ function resolvePleActivationStaticScaleValue(value, label) {
78
+ if (value == null) {
79
+ return null;
80
+ }
81
+ const parsed = Number(value);
82
+ if (!Number.isFinite(parsed) || parsed <= 0) {
83
+ throw new Error(
84
+ `Gemma 4 per-layer projection activation static scale "${label}" must be > 0. ` +
85
+ `Got "${String(value)}".`
86
+ );
87
+ }
88
+ return parsed;
89
+ }
90
+
91
+ function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
92
+ const inputScale = resolvePleActivationStaticScaleValue(
93
+ perLayerInputWeights?.perLayerModelProjectionInputActivationStaticScale,
94
+ 'input'
95
+ );
96
+ const outputScale = resolvePleActivationStaticScaleValue(
97
+ perLayerInputWeights?.perLayerModelProjectionOutputActivationStaticScale,
98
+ 'output'
99
+ );
100
+ if (inputScale == null && outputScale == null) {
101
+ return null;
102
+ }
103
+ if (inputScale == null || outputScale == null) {
104
+ throw new Error(
105
+ 'Gemma 4 per-layer projection static activation quantization requires both input and output scales.'
106
+ );
107
+ }
108
+ return {
109
+ inputScale,
110
+ outputScale,
111
+ };
112
+ }
113
+
114
+ function getTensorElementCount(tensor, label) {
115
+ if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
116
+ throw new Error(`${label} requires a tensor with an explicit shape.`);
117
+ }
118
+ const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
119
+ if (!Number.isFinite(count) || count <= 0) {
120
+ throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
121
+ }
122
+ return count;
123
+ }
124
+
125
+ async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
126
+ const count = getTensorElementCount(tensor, label);
127
+ if (tensor.dtype === 'f32') {
128
+ return recorder
129
+ ? await recordActivationStaticQdq(recorder, tensor, scale, {
130
+ count,
131
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
132
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
133
+ })
134
+ : await runActivationStaticQdq(tensor, scale, {
135
+ count,
136
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
137
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
138
+ });
139
+ }
140
+ if (tensor.dtype !== 'f16') {
141
+ throw new Error(
142
+ `${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
143
+ `Got "${String(tensor.dtype)}".`
144
+ );
145
+ }
146
+
147
+ let widenedTensor = null;
148
+ let qdqTensor = null;
149
+ try {
150
+ widenedTensor = recorder
151
+ ? await recordCastF16ToF32(recorder, tensor)
152
+ : await castF16ToF32(tensor);
153
+ qdqTensor = recorder
154
+ ? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
155
+ count,
156
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
157
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
158
+ })
159
+ : await runActivationStaticQdq(widenedTensor, scale, {
160
+ count,
161
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
162
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
163
+ });
164
+ return recorder
165
+ ? await recordCastF32ToF16(recorder, qdqTensor)
166
+ : await castF32ToF16(qdqTensor);
167
+ } finally {
168
+ if (qdqTensor) {
169
+ releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
170
+ }
171
+ if (widenedTensor) {
172
+ releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
173
+ }
174
+ }
175
+ }
176
+
66
177
  export function resolveDensePleProjectionWeight(
67
178
  weight,
68
179
  label = 'per_layer_model_projection'
@@ -1229,6 +1340,8 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1229
1340
  if (projectionDevice == null) {
1230
1341
  throw new Error('Gemma 4 per-layer input projection requires an initialized GPU device.');
1231
1342
  }
1343
+ const pleActivationStaticQdq = resolvePleActivationStaticQdqConfig(perLayerInputWeights);
1344
+ let projectionInputTensor = inputEmbedsTensor;
1232
1345
 
1233
1346
  // Decode-path optimizations: coalesced PLE read + fused projection matmul.
1234
1347
  // Gated on numTokens === 1 (decode) and row-major embeddings (non-transpose).
@@ -1237,8 +1350,12 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1237
1350
  const embedTranspose = (hasSplitEmbeddingTables || useHotVocabularyTables) ? false : getEmbeddingTranspose(embedTokensPerLayer);
1238
1351
  const canFuseDecodeOps = numTokens === 1 && !embedTranspose && !hasRangeBackedProjection;
1239
1352
  const tokenIdsAreGpuBuffer = isGpuBufferInstance(tokenIds);
1240
- const decodeTokenId = canFuseDecodeOps && !tokenIdsAreGpuBuffer
1241
- ? Number(tokenIds[0])
1353
+
1354
+ const tokenIdHint = Number.isInteger(options.tokenIdHint) ? Number(options.tokenIdHint) : null;
1355
+ const decodeTokenId = canFuseDecodeOps
1356
+ ? (tokenIdsAreGpuBuffer
1357
+ ? (Number.isInteger(tokenIdHint) ? tokenIdHint : null)
1358
+ : Number(tokenIds[0]))
1242
1359
  : null;
1243
1360
 
1244
1361
  if (decodeTokenId != null) {
@@ -1310,55 +1427,110 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1310
1427
  // Produces [numTokens × totalPerLayerHiddenSize], then scales the full output and
1311
1428
  // extracts per-layer slices via GPU buffer copies.
1312
1429
  let fusedProjectionSlices = null;
1313
- if (canFuseDecodeOps) {
1314
- const fusedProjection = await doMatmul(
1315
- inputEmbedsTensor,
1316
- projectionWeight,
1317
- numTokens,
1318
- totalPerLayerHiddenSize,
1319
- config.hiddenSize,
1320
- {
1321
- transposeB: 'auto',
1322
- label: 'per_layer_fused_projection',
1323
- kernelPath: context.kernelPath ?? null,
1324
- role: 'per_layer_model_projection',
1325
- outputDtype: activationDtype,
1326
- },
1327
- recorder
1328
- );
1329
-
1330
- // One scale dispatch instead of numLayers
1331
- const scaledFused = recorder
1332
- ? await recordScale(recorder, fusedProjection, projectionScale, {
1333
- count: numTokens * totalPerLayerHiddenSize,
1334
- })
1335
- : await runScale(fusedProjection, projectionScale, {
1336
- count: numTokens * totalPerLayerHiddenSize,
1337
- });
1338
- releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1339
-
1340
- // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1341
- // Reuse cached slice buffers when available to avoid per-step pool churn.
1342
- const device = getDevice();
1343
- const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1344
- const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
1345
- fusedProjectionSlices = new Array(numLayers);
1346
- for (let l = 0; l < numLayers; l++) {
1347
- const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
1348
- encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
1349
- fusedProjectionSlices[l] = sliceBuf;
1350
- }
1351
- if (!recorder) {
1352
- device.queue.submit([encoder.finish()]);
1353
- }
1354
- releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1355
- }
1356
-
1357
1430
  const embedDtypeResolved = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
1358
1431
  dtype: perLayerEmbeddingDtype,
1359
1432
  });
1360
1433
 
1361
1434
  try {
1435
+ if (pleActivationStaticQdq) {
1436
+ projectionInputTensor = await applyPleActivationStaticQdq(
1437
+ inputEmbedsTensor,
1438
+ pleActivationStaticQdq.inputScale,
1439
+ recorder,
1440
+ decodeBuffers,
1441
+ 'Gemma 4 per-layer projection input'
1442
+ );
1443
+ }
1444
+
1445
+ if (canFuseDecodeOps) {
1446
+ let fusedProjection = null;
1447
+ let fusedProjectionForScale = null;
1448
+ let scaledFused = null;
1449
+ try {
1450
+ // Per-layer-model projection: precision island. Source rows decoded
1451
+ // from int4_per_row produce activations whose magnitude before
1452
+ // RMSNorm-mediated scale-down can exceed f16 max (~65504). When the
1453
+ // matmul kernel does not internally widen its accumulator (RDNA3 is
1454
+ // strict, Metal happens to widen), the f16 output saturates to Inf
1455
+ // and then NaN. Compute matmul + QDQ + scale in f32 so the dynamic
1456
+ // range fits, then narrow to activationDtype after the scale step
1457
+ // brings magnitudes back to ~O(1).
1458
+ fusedProjection = await doMatmul(
1459
+ projectionInputTensor,
1460
+ projectionWeight,
1461
+ numTokens,
1462
+ totalPerLayerHiddenSize,
1463
+ config.hiddenSize,
1464
+ {
1465
+ transposeB: 'auto',
1466
+ label: 'per_layer_fused_projection',
1467
+ kernelPath: context.kernelPath ?? null,
1468
+ role: 'per_layer_model_projection',
1469
+ outputDtype: 'f32',
1470
+ },
1471
+ recorder
1472
+ );
1473
+ fusedProjectionForScale = pleActivationStaticQdq
1474
+ ? await applyPleActivationStaticQdq(
1475
+ fusedProjection,
1476
+ pleActivationStaticQdq.outputScale,
1477
+ recorder,
1478
+ decodeBuffers,
1479
+ 'Gemma 4 per-layer projection output'
1480
+ )
1481
+ : fusedProjection;
1482
+ if (fusedProjectionForScale !== fusedProjection) {
1483
+ releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1484
+ fusedProjection = null;
1485
+ }
1486
+
1487
+ let scaledFusedF32 = recorder
1488
+ ? await recordScale(recorder, fusedProjectionForScale, projectionScale, {
1489
+ count: numTokens * totalPerLayerHiddenSize,
1490
+ })
1491
+ : await runScale(fusedProjectionForScale, projectionScale, {
1492
+ count: numTokens * totalPerLayerHiddenSize,
1493
+ });
1494
+ releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
1495
+ fusedProjectionForScale = null;
1496
+
1497
+ scaledFused = scaledFusedF32.dtype === activationDtype
1498
+ ? scaledFusedF32
1499
+ : await doCast(scaledFusedF32, activationDtype, recorder);
1500
+ if (scaledFused !== scaledFusedF32) {
1501
+ releaseOrTrack(recorder, scaledFusedF32.buffer, decodeBuffers);
1502
+ }
1503
+
1504
+ // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1505
+ // Reuse cached slice buffers when available to avoid per-step pool churn.
1506
+ const device = getDevice();
1507
+ const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1508
+ const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
1509
+ fusedProjectionSlices = new Array(numLayers);
1510
+ for (let l = 0; l < numLayers; l++) {
1511
+ const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
1512
+ encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
1513
+ fusedProjectionSlices[l] = sliceBuf;
1514
+ }
1515
+ if (!recorder) {
1516
+ device.queue.submit([encoder.finish()]);
1517
+ }
1518
+ releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1519
+ scaledFused = null;
1520
+ } catch (error) {
1521
+ if (scaledFused) {
1522
+ releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1523
+ }
1524
+ if (fusedProjectionForScale && fusedProjectionForScale !== fusedProjection) {
1525
+ releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
1526
+ }
1527
+ if (fusedProjection) {
1528
+ releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1529
+ }
1530
+ throw error;
1531
+ }
1532
+ }
1533
+
1362
1534
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
1363
1535
  const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
1364
1536
  const layerEmbedSource = useHotVocabularyTables
@@ -1374,6 +1546,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1374
1546
  hiddenSize: hiddenSizePerLayerInput,
1375
1547
  vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
1376
1548
  scaleEmbeddings: true,
1549
+ probeStage: 'per_layer_embed_out',
1377
1550
  recorder,
1378
1551
  numTokens,
1379
1552
  indexOffset: perLayerTokenIds ? perLayerIndexOffset : indexOffset,
@@ -1390,6 +1563,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1390
1563
  outputBuffer: canFuseDecodeOps
1391
1564
  ? (pleCache?.gatherSliceBuffers?.[layerIdx] ?? undefined)
1392
1565
  : undefined,
1566
+ stats: context.stats ?? null,
1393
1567
  });
1394
1568
 
1395
1569
  if (fusedProjectionSlices) {
@@ -1406,6 +1580,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1406
1580
  let projectionWeightBufferForLayer = null;
1407
1581
  let projectionWeightOffset = 0;
1408
1582
  let projectedTensor = null;
1583
+ let projectionTensorForScale = null;
1409
1584
  try {
1410
1585
  const rangeBackedProjection = await loadRangeBackedPleProjectionSliceBytes(
1411
1586
  perLayerModelProjection,
@@ -1448,8 +1623,12 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1448
1623
  });
1449
1624
  projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
1450
1625
  }
1626
+ // Per-layer-model projection: precision island (see fused path
1627
+ // above for rationale). Force f32 through matmul + QDQ + scale,
1628
+ // then narrow to activationDtype after scale brings magnitudes
1629
+ // back to ~O(1).
1451
1630
  projectedTensor = await doMatmul(
1452
- inputEmbedsTensor,
1631
+ projectionInputTensor,
1453
1632
  projectionWeightForLayer,
1454
1633
  numTokens,
1455
1634
  hiddenSizePerLayerInput,
@@ -1461,24 +1640,70 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1461
1640
  layerIdx,
1462
1641
  kernelPath: context.kernelPath ?? null,
1463
1642
  role: 'per_layer_model_projection',
1464
- outputDtype: activationDtype,
1643
+ outputDtype: 'f32',
1465
1644
  },
1466
1645
  recorder
1467
1646
  );
1647
+ await runProbes('per_layer_projection_in', projectedTensor.buffer, {
1648
+ layerIdx,
1649
+ numTokens,
1650
+ hiddenSize: hiddenSizePerLayerInput,
1651
+ probes: context.debugProbes,
1652
+ recorder,
1653
+ operatorDiagnostics: context.operatorDiagnostics,
1654
+ dtype: 'f32',
1655
+ });
1656
+ projectionTensorForScale = pleActivationStaticQdq
1657
+ ? await applyPleActivationStaticQdq(
1658
+ projectedTensor,
1659
+ pleActivationStaticQdq.outputScale,
1660
+ recorder,
1661
+ decodeBuffers,
1662
+ `Gemma 4 per-layer projection output L${layerIdx}`
1663
+ )
1664
+ : projectedTensor;
1665
+ if (projectionTensorForScale !== projectedTensor) {
1666
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1667
+ projectedTensor = null;
1668
+ }
1669
+ const scaledF32 = recorder
1670
+ ? await recordScale(recorder, projectionTensorForScale, projectionScale, {
1671
+ count: numTokens * hiddenSizePerLayerInput,
1672
+ })
1673
+ : await runScale(projectionTensorForScale, projectionScale, {
1674
+ count: numTokens * hiddenSizePerLayerInput,
1675
+ });
1676
+ if (projectionTensorForScale === projectedTensor) {
1677
+ projectedTensor = null;
1678
+ }
1679
+ releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
1680
+ projectionTensorForScale = null;
1681
+ scaledProjectionTensor = scaledF32.dtype === activationDtype
1682
+ ? scaledF32
1683
+ : await doCast(scaledF32, activationDtype, recorder);
1684
+ if (scaledProjectionTensor !== scaledF32) {
1685
+ releaseOrTrack(recorder, scaledF32.buffer, decodeBuffers);
1686
+ }
1468
1687
  } finally {
1688
+ if (projectionTensorForScale && projectionTensorForScale !== projectedTensor) {
1689
+ releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
1690
+ }
1691
+ if (projectedTensor) {
1692
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1693
+ }
1469
1694
  if (projectionWeightBufferForLayer) {
1470
1695
  releaseOrTrack(recorder, projectionWeightBufferForLayer, decodeBuffers);
1471
1696
  }
1472
1697
  }
1473
- scaledProjectionTensor = recorder
1474
- ? await recordScale(recorder, projectedTensor, projectionScale, {
1475
- count: numTokens * hiddenSizePerLayerInput,
1476
- })
1477
- : await runScale(projectedTensor, projectionScale, {
1478
- count: numTokens * hiddenSizePerLayerInput,
1479
- });
1480
- releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1481
- projectedTensor = null;
1698
+ await runProbes('per_layer_projection_scaled', scaledProjectionTensor.buffer, {
1699
+ layerIdx,
1700
+ numTokens,
1701
+ hiddenSize: hiddenSizePerLayerInput,
1702
+ probes: context.debugProbes,
1703
+ recorder,
1704
+ operatorDiagnostics: context.operatorDiagnostics,
1705
+ dtype: activationDtype,
1706
+ });
1482
1707
  }
1483
1708
 
1484
1709
  // Fuse the residual add into RMSNorm so Gemma 4 PLE decode avoids an
@@ -1552,6 +1777,9 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1552
1777
  }
1553
1778
  throw error;
1554
1779
  } finally {
1780
+ if (projectionInputTensor !== inputEmbedsTensor) {
1781
+ releaseOrTrack(recorder, projectionInputTensor.buffer, decodeBuffers);
1782
+ }
1555
1783
  if (!usesCachedScaledProjectionNormWeight && !isGpuBufferInstance(perLayerProjectionNorm)) {
1556
1784
  releaseOrTrack(recorder, projectionNormWeight, decodeBuffers);
1557
1785
  }
@@ -1568,6 +1796,66 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1568
1796
  );
1569
1797
  }
1570
1798
 
1799
+ // Prefill hot-cache seeding: when we computed a multi-token batch and pleCache is
1800
+ // provided, extract per-token row slices and store unique token IDs in the cache.
1801
+ // Copies are recorded via the active recorder (if any) so they execute after the
1802
+ // batch computation and before the first decode step reads from the cache.
1803
+ // Without a recorder, the batch buffers are already populated so copies are safe
1804
+ // to submit immediately.
1805
+ if (pleCache) {
1806
+ const plePolicy = getPleHotCachePolicy(context.perLayerInputsSession ?? null);
1807
+ if (plePolicy && plePolicy.mode === 'prepared_tokens'
1808
+ && pleCache.preparedTokenEntries instanceof Map
1809
+ && !tokenIdsAreGpuBuffer
1810
+ && numTokens > 0
1811
+ ) {
1812
+ const tokenIdArray = Array.isArray(tokenIds) ? tokenIds : Array.from(tokenIds);
1813
+ const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1814
+ const seen = new Set();
1815
+ const device = getDevice();
1816
+ let pendingEncoder = null;
1817
+ for (let tokenPos = 0; tokenPos < tokenIdArray.length; tokenPos++) {
1818
+ const tid = tokenIdArray[tokenPos];
1819
+ if (seen.has(tid) || pleCache.preparedTokenEntries.has(tid)) continue;
1820
+ if (pleCache.preparedTokenEntries.size >= plePolicy.maxTokens) break;
1821
+ seen.add(tid);
1822
+ const srcOffset = tokenPos * sliceBytes;
1823
+ const sliceBuffers = new Array(numLayers).fill(null);
1824
+ let allValid = true;
1825
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
1826
+ const srcBuffer = perLayerBuffers[layerIdx];
1827
+ if (!srcBuffer || srcBuffer.size < srcOffset + sliceBytes) {
1828
+ allValid = false;
1829
+ break;
1830
+ }
1831
+ const dstBuffer = acquireBuffer(sliceBytes, undefined, `ple_seed_L${layerIdx}`);
1832
+ if (recorder) {
1833
+ recorder.getEncoder().copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
1834
+ } else {
1835
+ if (!pendingEncoder) pendingEncoder = device.createCommandEncoder();
1836
+ pendingEncoder.copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
1837
+ }
1838
+ sliceBuffers[layerIdx] = dstBuffer;
1839
+ }
1840
+ if (!allValid) {
1841
+ for (const buf of sliceBuffers) {
1842
+ if (buf) releaseBuffer(buf);
1843
+ }
1844
+ continue;
1845
+ }
1846
+ storePreparedTokenEntry(
1847
+ pleCache, tid, sliceBuffers,
1848
+ context.perLayerInputsSession ?? null,
1849
+ activationDtype,
1850
+ context.stats ?? null
1851
+ );
1852
+ }
1853
+ if (pendingEncoder && device) {
1854
+ device.queue.submit([pendingEncoder.finish()]);
1855
+ }
1856
+ }
1857
+ }
1858
+
1571
1859
  return perLayerBuffers;
1572
1860
  }
1573
1861
 
@@ -20,12 +20,9 @@ export function runProbes(
20
20
  hiddenSize: number;
21
21
  probes?: ProbeConfigSchema[] | null;
22
22
  recorder?: CommandRecorder | null;
23
+ operatorDiagnostics?: Record<string, unknown> | null;
23
24
  dtype?: 'f16' | 'f32';
25
+ phase?: string | null;
26
+ tokenIndex?: number | null;
24
27
  }
25
28
  ): Promise<void>;
26
-
27
- export function hasProbeStage(
28
- probes: ProbeConfigSchema[] | null | undefined,
29
- stage: ProbeStage,
30
- layerIdx?: number
31
- ): boolean;