doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -3,6 +3,7 @@
3
3
  import { trace } from '../../../debug/index.js';
4
4
  import { resolveCapturePolicy } from '../../../debug/capture-policy.js';
5
5
  import { snapshotTensor, snapshotFromArray } from '../../../debug/tensor.js';
6
+ import { computeArrayStats } from '../../../debug/stats.js';
6
7
  import { getDevice } from '../../../gpu/device.js';
7
8
  import { allowReadback } from '../../../gpu/perf-guards.js';
8
9
  import { f16ToF32 } from '../../../loader/dtype-utils.js';
@@ -11,10 +12,12 @@ import { PROBE_TO_CANONICAL } from './stage-names.js';
11
12
  import { buildOpId } from './operator-identity.js';
12
13
  import { getOperatorClass } from './stage-names.js';
13
14
  import { getDriftPolicyId } from './drift-policy.js';
15
+ import { maybeWriteFixtureSnapshot } from './tsir-fixture-writer.js';
14
16
 
15
17
 
16
18
  const STAGE_DEFAULT_CATEGORY = {
17
19
  embed_out: 'embed',
20
+ per_layer_embed_out: 'embed',
18
21
  // Attention stages (per-layer)
19
22
  attn_input: 'attn',
20
23
  post_input_norm: 'attn',
@@ -29,13 +32,17 @@ const STAGE_DEFAULT_CATEGORY = {
29
32
  v_proj: 'attn',
30
33
  q_norm: 'attn',
31
34
  k_norm: 'attn',
35
+ v_norm: 'attn',
32
36
  q_rope: 'attn',
33
37
  k_rope: 'attn',
34
38
  attn_scores: 'attn',
39
+ attn_core_out: 'attn',
35
40
  attn_out: 'attn',
36
41
  o_proj: 'attn',
37
42
  post_attn: 'attn',
38
43
  // FFN stages (per-layer)
44
+ per_layer_projection_in: 'ffn',
45
+ per_layer_projection_scaled: 'ffn',
39
46
  per_layer_input: 'ffn',
40
47
  per_layer_input_gate: 'ffn',
41
48
  per_layer_input_activation: 'ffn',
@@ -107,17 +114,105 @@ function getTraceLogger(category, layerIdx) {
107
114
  }
108
115
  }
109
116
 
117
+ function formatProbeNumber(value) {
118
+ return Number.isFinite(value) ? value.toFixed(4) : String(value);
119
+ }
120
+
121
+ function alignUp4(value) {
122
+ return Math.ceil(value / 4) * 4;
123
+ }
124
+
125
+ async function readGpuScalar(buffer, elementOffset, dtype, bytesPerElement) {
126
+ const byteOffset = elementOffset * bytesPerElement;
127
+ const alignedOffset = Math.floor(byteOffset / 4) * 4;
128
+ const offsetWithinRead = byteOffset - alignedOffset;
129
+ const readback = await readBufferSlice(buffer, alignedOffset, 4);
130
+ if (dtype === 'f16') {
131
+ const u16Array = new Uint16Array(readback);
132
+ return f16ToF32(u16Array[offsetWithinRead / 2]);
133
+ }
134
+ return new Float32Array(readback, offsetWithinRead, 1)[0];
135
+ }
136
+
137
+ async function readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer) {
138
+ if (isCpuBuffer) {
139
+ const start = tokenIdx * hiddenSize;
140
+ const end = start + hiddenSize;
141
+ return buffer.slice(start, end);
142
+ }
143
+
144
+ const bytesPerElement = dtype === 'f16' ? 2 : 4;
145
+ const byteOffset = tokenIdx * hiddenSize * bytesPerElement;
146
+ const rowBytes = hiddenSize * bytesPerElement;
147
+ const alignedOffset = Math.floor(byteOffset / 4) * 4;
148
+ const offsetWithinRead = byteOffset - alignedOffset;
149
+ const readSize = alignUp4(offsetWithinRead + rowBytes);
150
+ const readback = await readBufferSlice(buffer, alignedOffset, readSize);
151
+ const values = new Float32Array(hiddenSize);
152
+
153
+ if (dtype === 'f16') {
154
+ const u16Array = new Uint16Array(readback);
155
+ const start = offsetWithinRead / 2;
156
+ for (let i = 0; i < hiddenSize; i++) {
157
+ values[i] = f16ToF32(u16Array[start + i]);
158
+ }
159
+ return values;
160
+ }
161
+
162
+ const f32Array = new Float32Array(readback, offsetWithinRead, hiddenSize);
163
+ values.set(f32Array);
164
+ return values;
165
+ }
166
+
167
+ function formatProbeStats(stats) {
168
+ return [
169
+ `min=${formatProbeNumber(stats.min)}`,
170
+ `max=${formatProbeNumber(stats.max)}`,
171
+ `mean=${formatProbeNumber(stats.mean)}`,
172
+ `std=${formatProbeNumber(stats.std)}`,
173
+ `maxAbs=${formatProbeNumber(stats.maxAbs)}`,
174
+ `valid=${stats.validCount}`,
175
+ `nan=${stats.nanCount}`,
176
+ `inf=${stats.infCount}`,
177
+ `zero=${stats.zeroCount}`,
178
+ ].join(', ');
179
+ }
180
+
110
181
 
111
182
  export async function runProbes(stage, buffer, options) {
112
183
  const { layerIdx, numTokens, hiddenSize, probes, recorder, dtype = 'f32' } = options;
113
184
  if (!buffer) return;
114
- if (recorder && !options.operatorDiagnostics?.enabled) return;
185
+ // Skip when a recorder is in flight unless operatorDiagnostics is
186
+ // actively capturing (diagnose mode emitter or tsirFixture writer).
187
+ if (recorder
188
+ && !options.operatorDiagnostics?.enabled
189
+ && !options.operatorDiagnostics?.tsirFixture?.dir) return;
115
190
 
116
191
  const isCpuBuffer = buffer instanceof Float32Array;
117
192
  const device = isCpuBuffer ? null : getDevice();
118
193
  if (!isCpuBuffer && !device) return;
119
194
 
120
195
  const diagnostics = options.operatorDiagnostics ?? null;
196
+ // TSIR fixture writer: when operatorDiagnostics.tsirFixture.dir is
197
+ // set and the stage maps to one of the four TSIR boundary points
198
+ // (post_input_norm/linear_qkv_proj/post_attn/layer_out), capture
199
+ // the activation tensor as a .npy file under
200
+ // <dir>/layer_<layerIdx>/<tsirStage>.npy. Builds the rung-5 frozen
201
+ // Doppler reference fixture that the Doe rung ladder consumes.
202
+ if (diagnostics?.tsirFixture?.dir) {
203
+ const tsirRecord = await maybeWriteFixtureSnapshot(stage, buffer, {
204
+ tsirFixture: diagnostics.tsirFixture,
205
+ layerIdx,
206
+ numTokens,
207
+ hiddenSize,
208
+ dtype,
209
+ recorder,
210
+ });
211
+ if (tsirRecord) {
212
+ const recordList = (diagnostics.tsirFixture.records ??= []);
213
+ recordList.push(tsirRecord);
214
+ }
215
+ }
121
216
  const canonicalStage = getCanonicalStageName(stage);
122
217
  if (diagnostics?.enabled && diagnostics.emitter && canonicalStage) {
123
218
  const opId = buildOpId(canonicalStage, layerIdx);
@@ -127,6 +222,7 @@ export async function runProbes(stage, buffer, options) {
127
222
  isCpuBuffer,
128
223
  shape: [numTokens, hiddenSize],
129
224
  dtype,
225
+ recorder,
130
226
  });
131
227
  diagnostics.emitter.emitRecord(canonicalStage, {
132
228
  layerIdx,
@@ -149,14 +245,14 @@ export async function runProbes(stage, buffer, options) {
149
245
  if (stageProbes.length === 0) return;
150
246
  if (!isCpuBuffer && !allowReadback(`probe.${stage}`)) return;
151
247
 
152
- // Determine bytes per element based on dtype
153
248
  const bytesPerElement = dtype === 'f16' ? 2 : 4;
154
249
 
155
250
  for (const probe of stageProbes) {
156
251
  if (!matchesLayer(probe.layers, layerIdx)) continue;
157
252
 
158
- const dims = probe.dims ?? [];
159
- if (dims.length === 0) continue;
253
+ const dims = Array.isArray(probe.dims) ? probe.dims : [];
254
+ const includeStats = probe.stats === true;
255
+ if (dims.length === 0 && !includeStats) continue;
160
256
 
161
257
  const tokens = resolveTokens(probe.tokens, numTokens);
162
258
  if (tokens.length === 0) continue;
@@ -169,6 +265,12 @@ export async function runProbes(stage, buffer, options) {
169
265
 
170
266
  for (const tokenIdx of tokens) {
171
267
 
268
+ let statsText = null;
269
+ if (includeStats) {
270
+ const row = await readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer);
271
+ statsText = formatProbeStats(computeArrayStats(row));
272
+ }
273
+
172
274
  const values = [];
173
275
  for (const dimIdx of dims) {
174
276
  if (dimIdx < 0 || dimIdx >= hiddenSize) {
@@ -182,47 +284,42 @@ export async function runProbes(stage, buffer, options) {
182
284
  continue;
183
285
  }
184
286
  const elementOffset = tokenIdx * hiddenSize + dimIdx;
185
- const byteOffset = elementOffset * bytesPerElement;
186
- // WebGPU requires offset and size to be multiples of 4
187
- const alignedOffset = Math.floor(byteOffset / 4) * 4;
188
- const offsetWithinRead = byteOffset - alignedOffset;
189
- const readSize = 4; // Always read 4 bytes (aligned)
190
- const readback = await readBufferSlice(buffer, alignedOffset, readSize);
191
- let value;
192
- if (dtype === 'f16') {
193
- // offsetWithinRead is 0 or 2 for F16 - extract correct u16
194
- const u16Array = new Uint16Array(readback);
195
- const u16Index = offsetWithinRead / 2;
196
- value = f16ToF32(u16Array[u16Index]);
197
- } else {
198
- value = new Float32Array(readback)[0];
199
- }
200
- values.push(`${dimIdx}=${value.toFixed(4)}`);
287
+ const value = await readGpuScalar(buffer, elementOffset, dtype, bytesPerElement);
288
+ values.push(`${dimIdx}=${formatProbeNumber(value)}`);
201
289
  }
202
290
 
203
- logger(`PROBE${probeId} stage=${stage} token=${tokenIdx} values=[${values.join(', ')}]`);
291
+ const fields = [`PROBE${probeId} stage=${stage} token=${tokenIdx}`];
292
+ if (statsText) {
293
+ fields.push(`stats=[${statsText}]`);
294
+ }
295
+ if (values.length > 0) {
296
+ fields.push(`values=[${values.join(', ')}]`);
297
+ }
298
+ logger(fields.join(' '));
204
299
  }
205
300
  }
206
301
  }
207
302
 
208
303
 
209
- export function hasProbeStage(probes, stage, layerIdx) {
210
- if (!probes || probes.length === 0) return false;
211
- return probes.some((probe) => probe.stage === stage && matchesLayer(probe.layers, layerIdx));
212
- }
213
-
214
-
215
- export function getCanonicalStageName(probeStageName) {
304
+ function getCanonicalStageName(probeStageName) {
216
305
  return PROBE_TO_CANONICAL[probeStageName] ?? null;
217
306
  }
218
307
 
219
308
  async function buildDiagnosticCapture(level, buffer, options) {
220
309
  if (level === 'none') return null;
221
310
 
222
- const { isCpuBuffer, shape, dtype } = options;
311
+ const { isCpuBuffer, shape, dtype, recorder } = options;
312
+ const includeData = level === 'full';
313
+ if (!isCpuBuffer && recorder) {
314
+ return createDeferredDiagnosticCapture(level, buffer, {
315
+ recorder,
316
+ shape,
317
+ dtype,
318
+ });
319
+ }
223
320
  const snapshot = isCpuBuffer
224
- ? snapshotFromArray(buffer, shape, dtype)
225
- : await snapshotTensor(buffer, shape, dtype);
321
+ ? snapshotFromArray(buffer, shape, dtype, { includeData })
322
+ : await snapshotTensor(buffer, shape, dtype, { includeData });
226
323
  if (!snapshot?.ok) {
227
324
  return {
228
325
  level,
@@ -242,5 +339,75 @@ async function buildDiagnosticCapture(level, buffer, options) {
242
339
  stats: snapshot.stats ?? null,
243
340
  hasNaN: snapshot.hasNaN === true,
244
341
  hasInf: snapshot.hasInf === true,
342
+ data: Array.isArray(snapshot.data) ? snapshot.data : undefined,
343
+ };
344
+ }
345
+
346
+ function createDeferredDiagnosticCapture(level, buffer, options) {
347
+ const { recorder, shape, dtype } = options;
348
+ const elementSize = dtype === 'f16' ? 2 : 4;
349
+ const numElements = (shape ?? []).reduce((a, b) => a * b, 1);
350
+ const readSize = numElements > 0
351
+ ? Math.min(buffer.size, numElements * elementSize)
352
+ : buffer.size;
353
+ const staging = recorder.device.createBuffer({
354
+ label: `${recorder.label}_diagnostic_capture`,
355
+ size: readSize,
356
+ usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
357
+ });
358
+ recorder.getEncoder().copyBufferToBuffer(buffer, 0, staging, 0, readSize);
359
+
360
+ const capture = {
361
+ level,
362
+ shape,
363
+ dtype,
364
+ sample: null,
365
+ stats: null,
366
+ hasNaN: false,
367
+ hasInf: false,
245
368
  };
369
+
370
+ recorder.enqueueCompletionTask(async () => {
371
+ let mapped = false;
372
+ try {
373
+ await staging.mapAsync(GPUMapMode.READ);
374
+ mapped = true;
375
+ const snapshot = snapshotFromArray(
376
+ decodeSnapshotBytes(staging.getMappedRange().slice(0), dtype),
377
+ shape,
378
+ dtype,
379
+ { includeData: level === 'full' }
380
+ );
381
+ capture.shape = snapshot.shape;
382
+ capture.dtype = snapshot.dtype;
383
+ capture.sample = Array.isArray(snapshot.sample) ? snapshot.sample : null;
384
+ capture.stats = snapshot.stats ?? null;
385
+ capture.hasNaN = snapshot.hasNaN === true;
386
+ capture.hasInf = snapshot.hasInf === true;
387
+ capture.data = Array.isArray(snapshot.data) ? snapshot.data : undefined;
388
+ } catch (error) {
389
+ capture.error = error instanceof Error ? error.message : String(error);
390
+ capture.sample = null;
391
+ capture.stats = null;
392
+ } finally {
393
+ if (mapped) {
394
+ staging.unmap();
395
+ }
396
+ staging.destroy();
397
+ }
398
+ });
399
+
400
+ return capture;
401
+ }
402
+
403
+ function decodeSnapshotBytes(data, dtype) {
404
+ if (dtype === 'f16') {
405
+ const src = new Uint16Array(data);
406
+ const decoded = new Float32Array(src.length);
407
+ for (let i = 0; i < src.length; i++) {
408
+ decoded[i] = f16ToF32(src[i]);
409
+ }
410
+ return decoded;
411
+ }
412
+ return new Float32Array(data);
246
413
  }
@@ -11,11 +11,6 @@ export interface SamplingOptions {
11
11
  padTokenId?: number;
12
12
  }
13
13
 
14
- export interface TokenCandidate {
15
- token: number;
16
- prob: number;
17
- }
18
-
19
14
  export interface TopKResult {
20
15
  token: number;
21
16
  logit: number;
@@ -82,6 +82,139 @@ function assertFiniteSamplingCandidates(logits, padTokenId, label) {
82
82
  );
83
83
  }
84
84
 
85
+ function isHigherPriorityCandidate(candidate, current) {
86
+ return candidate.logit > current.logit
87
+ || (candidate.logit === current.logit && candidate.token < current.token);
88
+ }
89
+
90
+ function isLowerPriorityCandidate(candidate, current) {
91
+ return candidate.logit < current.logit
92
+ || (candidate.logit === current.logit && candidate.token > current.token);
93
+ }
94
+
95
+ function heapSwap(heap, a, b) {
96
+ const tmp = heap[a];
97
+ heap[a] = heap[b];
98
+ heap[b] = tmp;
99
+ }
100
+
101
+ function heapPushLowestFirst(heap, candidate) {
102
+ heap.push(candidate);
103
+ let index = heap.length - 1;
104
+ while (index > 0) {
105
+ const parent = Math.floor((index - 1) / 2);
106
+ if (!isLowerPriorityCandidate(heap[index], heap[parent])) {
107
+ break;
108
+ }
109
+ heapSwap(heap, index, parent);
110
+ index = parent;
111
+ }
112
+ }
113
+
114
+ function heapifyLowestFirst(heap, index) {
115
+ while (true) {
116
+ const left = index * 2 + 1;
117
+ const right = left + 1;
118
+ let smallest = index;
119
+ if (left < heap.length && isLowerPriorityCandidate(heap[left], heap[smallest])) {
120
+ smallest = left;
121
+ }
122
+ if (right < heap.length && isLowerPriorityCandidate(heap[right], heap[smallest])) {
123
+ smallest = right;
124
+ }
125
+ if (smallest === index) {
126
+ return;
127
+ }
128
+ heapSwap(heap, index, smallest);
129
+ index = smallest;
130
+ }
131
+ }
132
+
133
+ function selectTopKLogitCandidates(logits, topK) {
134
+ const limit = Math.max(1, Math.floor(topK));
135
+ if (!Number.isFinite(limit) || limit >= logits.length) {
136
+ return null;
137
+ }
138
+
139
+ const heap = [];
140
+ for (let token = 0; token < logits.length; token++) {
141
+ const logit = logits[token];
142
+ if (!Number.isFinite(logit)) {
143
+ continue;
144
+ }
145
+ const candidate = { token, logit };
146
+ if (heap.length < limit) {
147
+ heapPushLowestFirst(heap, candidate);
148
+ continue;
149
+ }
150
+ if (isHigherPriorityCandidate(candidate, heap[0])) {
151
+ heap[0] = candidate;
152
+ heapifyLowestFirst(heap, 0);
153
+ }
154
+ }
155
+ if (heap.length === 0) {
156
+ return [];
157
+ }
158
+ return heap.sort((a, b) => b.logit - a.logit || a.token - b.token);
159
+ }
160
+
161
+ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP) {
162
+ if (candidates.length === 0) {
163
+ throw new Error(
164
+ '[Sampling] Top-k filtering produced no finite candidates. ' +
165
+ 'Upstream decode likely produced NaN/Inf logits.'
166
+ );
167
+ }
168
+ if (candidates.length === 1) {
169
+ return candidates[0].token;
170
+ }
171
+
172
+ const invTemperature = 1 / temperature;
173
+ let maxScaled = -Infinity;
174
+ for (const candidate of candidates) {
175
+ const scaled = candidate.logit * invTemperature;
176
+ candidate.scaled = scaled;
177
+ if (scaled > maxScaled) {
178
+ maxScaled = scaled;
179
+ }
180
+ }
181
+
182
+ let sum = 0;
183
+ for (const candidate of candidates) {
184
+ const weight = Math.exp(candidate.scaled - maxScaled);
185
+ candidate.prob = weight;
186
+ sum += weight;
187
+ }
188
+
189
+ if (sum > 0) {
190
+ const invSum = 1 / sum;
191
+ for (const candidate of candidates) {
192
+ candidate.prob *= invSum;
193
+ }
194
+ } else {
195
+ const uniformProb = 1.0 / candidates.length;
196
+ for (const candidate of candidates) {
197
+ candidate.prob = uniformProb;
198
+ }
199
+ }
200
+
201
+ if (debug) {
202
+ const top5 = candidates.slice(0, 5).map(c => {
203
+ const text = decode?.([c.token]) ?? '?';
204
+ return `"${text}"(${(c.prob * 100).toFixed(1)}%)`;
205
+ });
206
+ trace.sample(`Top-5 (temp=${temperature}, topK=${topK}, topP=${topP}): ${top5.join(', ')}`);
207
+ }
208
+
209
+ const r = seed !== undefined ? seededRandom(seed) : unseededRandom();
210
+ let cumProb = 0;
211
+ for (const candidate of candidates) {
212
+ cumProb += candidate.prob;
213
+ if (r < cumProb) return candidate.token;
214
+ }
215
+ return candidates[candidates.length - 1].token;
216
+ }
217
+
85
218
 
86
219
  export function sample(logits, opts) {
87
220
  const { temperature, topP, topK, decode, debug = false, padTokenId, seed } = opts;
@@ -119,6 +252,13 @@ export function sample(logits, opts) {
119
252
  return maxIdx;
120
253
  }
121
254
 
255
+ if (topP >= 1.0 && Number.isFinite(topK) && topK > 0) {
256
+ const candidates = selectTopKLogitCandidates(logits, topK);
257
+ if (candidates) {
258
+ return sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP);
259
+ }
260
+ }
261
+
122
262
  // Apply temperature
123
263
  if (temperature !== 1.0) {
124
264
  for (let i = 0; i < logits.length; i++) {
@@ -36,6 +36,7 @@ export const OPERATOR_CLASSES = Object.freeze({
36
36
  export const STAGES = Object.freeze({
37
37
  // Embedding
38
38
  EMBED_OUT: 'embed.out',
39
+ PER_LAYER_EMBED_OUT: 'per_layer_embed.out',
39
40
 
40
41
  // Attention stages (per-layer, prefixed with attn.)
41
42
  ATTN_INPUT: 'attn.input',
@@ -51,6 +52,7 @@ export const STAGES = Object.freeze({
51
52
  ATTN_K_ROPE: 'attn.k_rope',
52
53
  ATTN_SCORES: 'attn.scores',
53
54
  ATTN_SOFTMAX: 'attn.softmax',
55
+ ATTN_CORE_OUT: 'attn.core_out',
54
56
  ATTN_OUT: 'attn.out',
55
57
  ATTN_O_PROJ: 'attn.o_proj',
56
58
  ATTN_POST_ATTN: 'attn.post_attn',
@@ -68,6 +70,8 @@ export const STAGES = Object.freeze({
68
70
  CONV_OUT_PROJ: 'conv.out_proj',
69
71
 
70
72
  // FFN stages (per-layer)
73
+ PER_LAYER_PROJECTION_IN: 'per_layer_projection.in',
74
+ PER_LAYER_PROJECTION_SCALED: 'per_layer_projection.scaled',
71
75
  PER_LAYER_INPUT_OUT: 'per_layer_input.out',
72
76
  FFN_NORMED: 'ffn.normed',
73
77
  FFN_IN: 'ffn.in',
@@ -100,6 +104,7 @@ export const STAGES = Object.freeze({
100
104
 
101
105
  const STAGE_TO_CLASS_MAP = Object.freeze({
102
106
  [STAGES.EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
107
+ [STAGES.PER_LAYER_EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
103
108
 
104
109
  [STAGES.ATTN_INPUT]: OPERATOR_CLASSES.NORMALIZATION,
105
110
  [STAGES.ATTN_POST_INPUT_NORM]: OPERATOR_CLASSES.NORMALIZATION,
@@ -114,6 +119,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
114
119
  [STAGES.ATTN_K_ROPE]: OPERATOR_CLASSES.ROPE,
115
120
  [STAGES.ATTN_SCORES]: OPERATOR_CLASSES.ATTENTION,
116
121
  [STAGES.ATTN_SOFTMAX]: OPERATOR_CLASSES.ATTENTION,
122
+ [STAGES.ATTN_CORE_OUT]: OPERATOR_CLASSES.ATTENTION,
117
123
  [STAGES.ATTN_OUT]: OPERATOR_CLASSES.ATTENTION,
118
124
  [STAGES.ATTN_O_PROJ]: OPERATOR_CLASSES.PROJECTION,
119
125
  [STAGES.ATTN_POST_ATTN]: OPERATOR_CLASSES.RESIDUAL,
@@ -128,6 +134,8 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
128
134
  [STAGES.CONV_KERNEL]: OPERATOR_CLASSES.CONV,
129
135
  [STAGES.CONV_OUT_PROJ]: OPERATOR_CLASSES.CONV,
130
136
 
137
+ [STAGES.PER_LAYER_PROJECTION_IN]: OPERATOR_CLASSES.PROJECTION,
138
+ [STAGES.PER_LAYER_PROJECTION_SCALED]: OPERATOR_CLASSES.PROJECTION,
131
139
  [STAGES.PER_LAYER_INPUT_OUT]: OPERATOR_CLASSES.RESIDUAL,
132
140
  [STAGES.FFN_NORMED]: OPERATOR_CLASSES.NORMALIZATION,
133
141
  [STAGES.FFN_IN]: OPERATOR_CLASSES.FFN,
@@ -155,6 +163,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
155
163
 
156
164
  export const PROBE_TO_CANONICAL = Object.freeze({
157
165
  embed_out: STAGES.EMBED_OUT,
166
+ per_layer_embed_out: STAGES.PER_LAYER_EMBED_OUT,
158
167
  attn_input: STAGES.ATTN_INPUT,
159
168
  post_input_norm: STAGES.ATTN_POST_INPUT_NORM,
160
169
  attn_normed: STAGES.ATTN_NORMED,
@@ -171,9 +180,12 @@ export const PROBE_TO_CANONICAL = Object.freeze({
171
180
  q_rope: STAGES.ATTN_Q_ROPE,
172
181
  k_rope: STAGES.ATTN_K_ROPE,
173
182
  attn_scores: STAGES.ATTN_SCORES,
183
+ attn_core_out: STAGES.ATTN_CORE_OUT,
174
184
  attn_out: STAGES.ATTN_OUT,
175
185
  o_proj: STAGES.ATTN_O_PROJ,
176
186
  post_attn: STAGES.ATTN_POST_ATTN,
187
+ per_layer_projection_in: STAGES.PER_LAYER_PROJECTION_IN,
188
+ per_layer_projection_scaled: STAGES.PER_LAYER_PROJECTION_SCALED,
177
189
  per_layer_input: STAGES.PER_LAYER_INPUT_OUT,
178
190
  ffn_normed: STAGES.FFN_NORMED,
179
191
  ffn_in: STAGES.FFN_IN,
@@ -102,6 +102,8 @@ export class PipelineState {
102
102
  decodeRecordMs: 0,
103
103
  decodeSubmitWaitMs: 0,
104
104
  decodeReadbackWaitMs: 0,
105
+ prefillRecordMs: 0,
106
+ prefillSubmitWaitMs: 0,
105
107
  prefillProfileSteps: [],
106
108
  decodeProfileSteps: [],
107
109
  executionPlan: null,