doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,11 +1,14 @@
1
1
 
2
2
 
3
3
  import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { acquireBuffer } from '../../../../memory/buffer-pool.js';
4
+ import { getKernelCapabilities } from '../../../../gpu/device.js';
5
+ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
5
6
  import {
6
7
  recordMatmul,
7
8
  recordRMSNorm,
8
9
  recordRoPE,
10
+ canUseRoPEQK,
11
+ recordRoPEQK,
9
12
  recordAttention,
10
13
  recordAttentionTiered,
11
14
  recordAttentionTieredQuant,
@@ -20,7 +23,7 @@ import {
20
23
  import { createTensor } from '../../../../gpu/tensor.js';
21
24
  import { applyLoRA } from '../lora-apply.js';
22
25
  import { getLoRAModule } from '../lora.js';
23
- import { log, trace } from '../../../../debug/index.js';
26
+ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
24
27
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
25
28
  import {
26
29
  recordAttentionInputs,
@@ -29,8 +32,13 @@ import {
29
32
  projectAttentionQKV,
30
33
  applyAttentionQKNorm,
31
34
  applyAttentionValueNorm,
35
+ hasAttentionProjectionDiagnostics,
36
+ hasAttentionStageDiagnostics,
37
+ resolveAttentionQKNormState,
32
38
  } from './projections.js';
33
39
  import { prepareAttentionProjectionInput } from './output-projection.js';
40
+ import { runProbes } from '../probes.js';
41
+ import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
34
42
 
35
43
  import { releaseOrTrack, shouldDebugLayer } from './types.js';
36
44
  import {
@@ -39,6 +47,7 @@ import {
39
47
  } from '../../../../config/kernel-path-loader.js';
40
48
  import {
41
49
  resolveKVCacheState,
50
+ createDiffusionGemmaDecoderKVState,
42
51
  buildAttentionDispatchParams,
43
52
  buildAttentionInputsData,
44
53
  } from './dispatch-params.js';
@@ -47,13 +56,102 @@ import {
47
56
  buildContiguousQuantAttentionOptions,
48
57
  } from './quant-options.js';
49
58
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
59
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
50
60
  import {
51
61
  resolveAttentionPrecisionContract,
52
62
  isAttentionKvDtypeExplicit,
53
63
  } from './precision-contract.js';
64
+ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
54
65
 
55
66
  const ATTENTION_DTYPE_LOGGED = new Set();
56
67
 
68
+ function isWideTileQ4KPhaseEnabled(session, phase) {
69
+ return phase === 'decode'
70
+ ? session?.useWideTileQ4KDecode === true
71
+ : session?.useWideTileQ4KPrefill === true;
72
+ }
73
+
74
+ function canUseQ4KWideTileResidualFusion(options = {}) {
75
+ return options.hasResidual === true
76
+ && options.oProjDtype === 'q4k'
77
+ && options.inputDtype === 'f32'
78
+ && options.outputDtype === 'f32'
79
+ && options.residualMatches === true
80
+ && options.hasLoRA !== true
81
+ && getKernelCapabilities().hasF16 === true
82
+ && getKernelCapabilities().hasSubgroups === true
83
+ && options.session?.useWideTileResidualFusion === true
84
+ && options.session?.retainQ4KMaterialization === true
85
+ && isWideTileQ4KPhaseEnabled(options.session, options.phase);
86
+ }
87
+
88
+ function shouldTraceRecordedHealth(layerIdx, debugFlags) {
89
+ const debugLayers = debugFlags?.debugLayers;
90
+ return isTraceEnabled('logits')
91
+ && Array.isArray(debugLayers)
92
+ && shouldDebugLayer(layerIdx, debugLayers);
93
+ }
94
+
95
+ function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
96
+ if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
97
+ return;
98
+ }
99
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
100
+ recorder.enqueueCompletionTask(async () => {
101
+ const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
102
+ trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
103
+ });
104
+ }
105
+
106
+ function resolveDirectF16KVCacheWrite(options) {
107
+ const {
108
+ state,
109
+ layerIdx,
110
+ currentSeqLen,
111
+ numTokens,
112
+ numKVHeads,
113
+ headDim,
114
+ reusesSharedKV,
115
+ storeSharedKV,
116
+ diffusionGemmaDecoder,
117
+ valueNorm,
118
+ diagnosticsEnabled,
119
+ disableRoPE,
120
+ } = options;
121
+ if (
122
+ numTokens !== 1
123
+ || reusesSharedKV === true
124
+ || storeSharedKV === true
125
+ || diffusionGemmaDecoder === true
126
+ || valueNorm === true
127
+ || diagnosticsEnabled === true
128
+ || disableRoPE === true
129
+ ) {
130
+ return null;
131
+ }
132
+ const cache = state?.kvCache;
133
+ if (
134
+ !cache?.hasGPUCache?.()
135
+ || cache.kvDtype !== 'f16'
136
+ || cache.layout !== 'contiguous'
137
+ || cache.windowSize != null
138
+ || cache.layerSpecs != null
139
+ || typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
140
+ ) {
141
+ return null;
142
+ }
143
+ const gpuBuffers = cache.getGPUBuffers(layerIdx);
144
+ const layout = gpuBuffers?.layout ?? cache.layout;
145
+ if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
146
+ return null;
147
+ }
148
+ return {
149
+ keysBuffer: gpuBuffers.keysGPU,
150
+ valuesBuffer: gpuBuffers.valuesGPU,
151
+ dstOffset: currentSeqLen * numKVHeads * headDim,
152
+ };
153
+ }
154
+
57
155
  function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
58
156
  assertImplicitDtypeTransitionAllowed({
59
157
  executionPolicies: state?.executionPolicies ?? null,
@@ -101,6 +199,8 @@ export async function recordLayerAttentionGPU(
101
199
  multimodalBidirectionalSpan = null,
102
200
  sharedKVSourceLayerIdx = null,
103
201
  storeSharedKV = false,
202
+ diffusionGemmaDecoder = false,
203
+ precomputedInputNorm = null,
104
204
  } = config;
105
205
 
106
206
  const phase = isPrefill ? 'prefill' : 'decode';
@@ -122,6 +222,9 @@ export async function recordLayerAttentionGPU(
122
222
  const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
123
223
  const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
124
224
  const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
225
+ const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
226
+ ? 'step_precision'
227
+ : null;
125
228
  let attentionInput = input;
126
229
  let attentionInputTemp = false;
127
230
  let normed = attentionInput;
@@ -135,12 +238,17 @@ export async function recordLayerAttentionGPU(
135
238
  let finalOutput = null;
136
239
  let oProjInputTemp = null;
137
240
  let retainSharedKvBuffers = false;
241
+ let valueAliasesKey = false;
242
+ let decoderKVState = null;
243
+ let decoderKTemp = null;
244
+ let decoderVTemp = null;
138
245
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
139
246
  assertAttentionDtypeTransitionAllowed(
140
247
  state,
141
248
  input.dtype,
142
249
  attentionActivationDtype,
143
- 'The attention kernel selection would widen the input implicitly.'
250
+ 'The attention kernel selection would widen the input implicitly.',
251
+ attentionInputTransitionDeclaredBy
144
252
  );
145
253
  attentionInput = attentionActivationDtype === 'f16'
146
254
  ? await recordCastF32ToF16(recorder, input)
@@ -160,16 +268,68 @@ export async function recordLayerAttentionGPU(
160
268
  const kvSize = numTokens * numKVHeads * headDim;
161
269
 
162
270
  // 1. Input norm
271
+ // Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
272
+ // rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
273
+ let fusedNormWeightRec = null;
274
+ let fusedNormEpsRec = null;
275
+ let fusedNormOffsetRec = false;
276
+ let fusedNormOwnedRec = false;
277
+ // state.runtimeConfig is a stale snapshot whose inference.session is an empty
278
+ // object; the live module-level runtime config carries the merged
279
+ // profile/override session. Read session-level flags from getRuntimeConfig().
280
+ const rmsNormFusionFlagRec = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
281
+ const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
282
+ layerWeights,
283
+ sharedKVSourceLayerIdx != null
284
+ );
285
+ const canFuseInputNormProjRec = rmsNormFusionFlagRec
286
+ && canSelectFusedRmsNormProjectionRec
287
+ && !skipInputNorm
288
+ && layerWeights.inputNorm
289
+ && getNormWeightBuffer
290
+ && isPrefill
291
+ && numTokens > 1
292
+ && attentionInput.dtype === 'f32';
163
293
 
164
294
  try {
165
- if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
295
+ if (precomputedInputNorm) {
296
+ if (
297
+ isPrefill
298
+ || numTokens !== 1
299
+ || skipInputNorm
300
+ || !layerWeights.inputNorm
301
+ || !getNormWeightBuffer
302
+ || precomputedInputNorm.dtype !== attentionInput.dtype
303
+ || precomputedInputNorm.dtype !== 'f32'
304
+ ) {
305
+ throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
306
+ }
307
+ normed = precomputedInputNorm;
308
+ await runProbes('post_input_norm', normed.buffer, {
309
+ layerIdx, numTokens, hiddenSize,
310
+ probes: state.debugProbes, recorder,
311
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
312
+ });
313
+ } else if (canFuseInputNormProjRec) {
314
+ fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
315
+ fusedNormEpsRec = rmsNormEps;
316
+ fusedNormOffsetRec = config.rmsNormWeightOffset === true;
317
+ fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
318
+ // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
319
+ } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
166
320
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
167
321
  normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
168
322
  batchSize: numTokens,
169
323
  hiddenSize,
170
324
  rmsNormWeightOffset: config.rmsNormWeightOffset,
325
+ label: 'input_norm',
171
326
  });
172
327
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
328
+ await runProbes('post_input_norm', normed.buffer, {
329
+ layerIdx, numTokens, hiddenSize,
330
+ probes: state.debugProbes, recorder,
331
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
332
+ });
173
333
  }
174
334
 
175
335
  const debugLayers = debugFlags.debugLayers;
@@ -217,7 +377,35 @@ export async function recordLayerAttentionGPU(
217
377
  }
218
378
  const reusesSharedKV = sharedKVEntry != null;
219
379
  retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
220
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
380
+ let qkNormApplied = false;
381
+ let ropeApplied = false;
382
+ const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
383
+ const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
384
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
385
+ const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
386
+ state,
387
+ ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
388
+ )
389
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
390
+ const runtimeSession = getRuntimeConfig()?.inference?.session;
391
+ const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
392
+ const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
393
+ const directF16KVCacheWrite = resolveDirectF16KVCacheWrite({
394
+ state,
395
+ layerIdx,
396
+ currentSeqLen,
397
+ numTokens,
398
+ numKVHeads,
399
+ headDim,
400
+ reusesSharedKV,
401
+ storeSharedKV,
402
+ diffusionGemmaDecoder,
403
+ valueNorm: config.valueNorm === true,
404
+ diagnosticsEnabled: qkNormRoPEDiagnostics,
405
+ disableRoPE,
406
+ });
407
+ let kvCacheWriteFused = false;
408
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
221
409
  recorder,
222
410
  normed,
223
411
  layerWeights,
@@ -242,31 +430,146 @@ export async function recordLayerAttentionGPU(
242
430
  trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
243
431
  }
244
432
  : null,
433
+ fusedNormWeight: fusedNormWeightRec,
434
+ fusedNormEps: fusedNormEpsRec,
435
+ fusedNormOffset: fusedNormOffsetRec,
436
+ qkNormFusion: {
437
+ enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
438
+ getNormWeightBuffer,
439
+ rmsNormEps,
440
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
441
+ skipKNorm: qkNormState.skipKNorm,
442
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
443
+ projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
444
+ },
445
+ qkNormRoPEFusion: {
446
+ enabled: qkNormRoPEFusionFlag
447
+ && qkNormState.wantsQKNorm
448
+ && !disableRoPE
449
+ && !!state.ropeFreqsCos
450
+ && !!state.ropeFreqsSin,
451
+ getNormWeightBuffer,
452
+ rmsNormEps,
453
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
454
+ skipKNorm: qkNormState.skipKNorm,
455
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
456
+ projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
457
+ freqsCos: state.ropeFreqsCos,
458
+ freqsSin: state.ropeFreqsSin,
459
+ startPos: currentSeqLen,
460
+ rotaryDim: config.ropeRotaryDim,
461
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
462
+ interleaved: config.ropeInterleaved,
463
+ reusesSharedKV,
464
+ f16KVCacheWrite: directF16KVCacheWrite,
465
+ },
245
466
  }));
467
+ if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
468
+ throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
469
+ }
470
+ // Deferred release of the norm weight buffer for fused path.
471
+ if (fusedNormWeightRec && fusedNormOwnedRec) {
472
+ releaseOrTrack(recorder, fusedNormWeightRec);
473
+ }
474
+ if (qTensor) {
475
+ await runProbes('q_proj', qTensor.buffer, {
476
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
477
+ probes: state.debugProbes, recorder,
478
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
479
+ });
480
+ }
481
+ if (kTensor) {
482
+ await runProbes('k_proj', kTensor.buffer, {
483
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
484
+ probes: state.debugProbes, recorder,
485
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
486
+ });
487
+ }
488
+ if (vTensor) {
489
+ await runProbes('v_proj', vTensor.buffer, {
490
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
491
+ probes: state.debugProbes, recorder,
492
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
493
+ });
494
+ }
495
+ if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
496
+ enqueueRecordedTensorHealth(
497
+ recorder,
498
+ `L${layerIdx}.q_proj_HEALTH`,
499
+ qTensor,
500
+ qTensor.dtype,
501
+ numTokens * numHeads * headDim
502
+ );
503
+ enqueueRecordedTensorHealth(
504
+ recorder,
505
+ `L${layerIdx}.k_proj_HEALTH`,
506
+ kTensor,
507
+ kTensor?.dtype ?? null,
508
+ numTokens * numKVHeads * headDim
509
+ );
510
+ enqueueRecordedTensorHealth(
511
+ recorder,
512
+ `L${layerIdx}.v_proj_HEALTH`,
513
+ vTensor,
514
+ vTensor?.dtype ?? null,
515
+ numTokens * numKVHeads * headDim
516
+ );
517
+ }
246
518
 
247
519
  // Optional per-head Q/K normalization.
248
520
  // Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
249
- const wantsQKNorm = config.queryKeyNorm === true;
250
- if (wantsQKNorm && layerIdx === 0 && (!layerWeights.qNorm || !layerWeights.kNorm)) {
251
- log.warn('Attention', `Q/K norm requested but weights missing (hasQ=${!!layerWeights.qNorm}, hasK=${!!layerWeights.kNorm}); skipping QK norm.`);
521
+ if (qkNormState.wantsQKNorm && !qkNormApplied) {
522
+ ({ qTensor, kTensor } = await applyAttentionQKNorm({
523
+ recorder,
524
+ qTensor,
525
+ kTensor,
526
+ layerWeights,
527
+ getNormWeightBuffer,
528
+ rmsNormEps,
529
+ numTokens,
530
+ numHeads,
531
+ numKVHeads,
532
+ headDim,
533
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
534
+ releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
535
+ skipKNorm: qkNormState.skipKNorm,
536
+ retainKInput: valueAliasesKey,
537
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
538
+ }));
539
+ }
540
+ if (qkNormState.wantsQKNorm) {
541
+ await runProbes('q_norm', qTensor.buffer, {
542
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
543
+ probes: state.debugProbes, recorder,
544
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
545
+ });
546
+ if (kTensor) {
547
+ await runProbes('k_norm', kTensor.buffer, {
548
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
549
+ probes: state.debugProbes, recorder,
550
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
551
+ });
552
+ }
553
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
554
+ enqueueRecordedTensorHealth(
555
+ recorder,
556
+ `L${layerIdx}.q_norm_HEALTH`,
557
+ qTensor,
558
+ qTensor.dtype,
559
+ numTokens * numHeads * headDim
560
+ );
561
+ enqueueRecordedTensorHealth(
562
+ recorder,
563
+ `L${layerIdx}.k_norm_HEALTH`,
564
+ kTensor,
565
+ kTensor?.dtype ?? null,
566
+ numTokens * numKVHeads * headDim
567
+ );
568
+ }
252
569
  }
253
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
254
- recorder,
255
- qTensor,
256
- kTensor,
257
- layerWeights,
258
- getNormWeightBuffer,
259
- rmsNormEps,
260
- numTokens,
261
- numHeads,
262
- numKVHeads,
263
- headDim,
264
- rmsNormWeightOffset: config.rmsNormWeightOffset,
265
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
266
- skipKNorm: reusesSharedKV,
267
- }));
268
570
 
269
- if (config.valueNorm === true && !reusesSharedKV) {
571
+ if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
572
+ const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
270
573
  vTensor = await applyAttentionValueNorm({
271
574
  recorder,
272
575
  vTensor,
@@ -274,7 +577,16 @@ export async function recordLayerAttentionGPU(
274
577
  numTokens,
275
578
  numKVHeads,
276
579
  headDim,
277
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
580
+ releaseTemporary: (buffer) => {
581
+ if (!valueNormInputAliasesKey) {
582
+ releaseOrTrack(recorder, buffer);
583
+ }
584
+ },
585
+ });
586
+ await runProbes('v_norm', vTensor.buffer, {
587
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
588
+ probes: state.debugProbes, recorder,
589
+ operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
278
590
  });
279
591
  }
280
592
 
@@ -282,26 +594,50 @@ export async function recordLayerAttentionGPU(
282
594
  if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
283
595
 
284
596
  // 3. RoPE (modifies tensor in-place)
285
- if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
286
- await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
287
- numHeads,
597
+ if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
598
+ const ropeOptions = {
288
599
  headDim,
289
600
  rotaryDim: config.ropeRotaryDim,
601
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
290
602
  interleaved: config.ropeInterleaved,
291
603
  startPos: currentSeqLen,
292
604
  executionPolicies: state.executionPolicies ?? null,
293
- });
294
- if (!reusesSharedKV) {
295
- await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
296
- numHeads: numKVHeads,
297
- headDim,
298
- rotaryDim: config.ropeRotaryDim,
299
- interleaved: config.ropeInterleaved,
300
- startPos: currentSeqLen,
301
- executionPolicies: state.executionPolicies ?? null,
605
+ };
606
+ if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
607
+ await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
608
+ ...ropeOptions,
609
+ numQHeads: numHeads,
610
+ numKVHeads,
611
+ });
612
+ } else {
613
+ await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
614
+ ...ropeOptions,
615
+ numHeads,
302
616
  });
617
+ if (!reusesSharedKV) {
618
+ await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
619
+ ...ropeOptions,
620
+ numHeads: numKVHeads,
621
+ });
622
+ }
303
623
  }
304
624
  }
625
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
626
+ enqueueRecordedTensorHealth(
627
+ recorder,
628
+ `L${layerIdx}.q_rope_HEALTH`,
629
+ qTensor,
630
+ qTensor.dtype,
631
+ numTokens * numHeads * headDim
632
+ );
633
+ enqueueRecordedTensorHealth(
634
+ recorder,
635
+ `L${layerIdx}.k_rope_HEALTH`,
636
+ kTensor,
637
+ kTensor?.dtype ?? null,
638
+ numTokens * numKVHeads * headDim
639
+ );
640
+ }
305
641
 
306
642
  if (storeSharedKV && state.sharedAttentionState) {
307
643
  state.sharedAttentionState.set(layerIdx, {
@@ -313,10 +649,10 @@ export async function recordLayerAttentionGPU(
313
649
  }
314
650
 
315
651
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
316
- if (state.kvCache?.hasGPUCache?.()) {
317
- // Use recordUpdateFromGPU to record copy operations to the recorder's encoder
318
- // This ensures K/V buffers are populated before copying (all ops submitted together)
319
- if (state.kvCache.kvDtype === 'f16') {
652
+ if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
653
+ if (kvCacheWriteFused) {
654
+ state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
655
+ } else if (state.kvCache.kvDtype === 'f16') {
320
656
  const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
321
657
  if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
322
658
  assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
@@ -324,24 +660,95 @@ export async function recordLayerAttentionGPU(
324
660
  if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
325
661
  assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
326
662
  }
327
- const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
328
- const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
663
+ const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
664
+ && vTensor.dtype === 'f32'
665
+ && typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
666
+ if (canWriteDirectF32ToF16) {
667
+ await state.kvCache.recordUpdateF32ToF16FromGPU(
668
+ recorder,
669
+ layerIdx,
670
+ kTensor.buffer,
671
+ vTensor.buffer,
672
+ currentSeqLen,
673
+ numTokens,
674
+ tokenIds
675
+ );
676
+ } else {
677
+ const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
678
+ const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
329
679
 
330
- await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
680
+ await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
331
681
 
332
- if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
333
- if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
682
+ if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
683
+ if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
684
+ }
334
685
  } else {
335
686
  await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
336
687
  }
337
688
  }
338
689
 
339
690
  // Resolve KV cache state and build dispatch parameters (shared with run.js)
340
- const kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
691
+ let kvState;
692
+ if (diffusionGemmaDecoder) {
693
+ const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
694
+ let decoderKTensor = kTensor;
695
+ let decoderVTensor = vTensor;
696
+ if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
697
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
698
+ if (!hasExplicitF16KvContract) {
699
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
700
+ }
701
+ decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
702
+ decoderKTemp = decoderKTensor;
703
+ } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
704
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
705
+ decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
706
+ decoderKTemp = decoderKTensor;
707
+ }
708
+ if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
709
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
710
+ if (!hasExplicitF16KvContract) {
711
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
712
+ }
713
+ decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
714
+ decoderVTemp = decoderVTensor;
715
+ } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
716
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
717
+ decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
718
+ decoderVTemp = decoderVTensor;
719
+ }
720
+ kvState = await createDiffusionGemmaDecoderKVState({
721
+ state,
722
+ layerIdx,
723
+ kTensor: decoderKTensor,
724
+ vTensor: decoderVTensor,
725
+ currentSeqLen,
726
+ numTokens,
727
+ numKVHeads,
728
+ headDim,
729
+ layerType,
730
+ slidingWindow,
731
+ kvDtype: decoderKVDtype,
732
+ recorder,
733
+ });
734
+ if (decoderKTemp) {
735
+ recorder.trackTemporaryBuffer(decoderKTemp.buffer);
736
+ decoderKTemp = null;
737
+ }
738
+ if (decoderVTemp) {
739
+ recorder.trackTemporaryBuffer(decoderVTemp.buffer);
740
+ decoderVTemp = null;
741
+ }
742
+ decoderKVState = kvState;
743
+ } else {
744
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
745
+ }
341
746
  const dispatchConfig = {
342
747
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
343
- slidingWindow, layerType, layerTypes: config.layerTypes,
344
- queryPreAttnScalar, causalAttention: config.causalAttention,
748
+ slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
749
+ layerType, layerTypes: config.layerTypes,
750
+ queryPreAttnScalar,
751
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
345
752
  activationDtype: attentionActivationDtype,
346
753
  kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
347
754
  };
@@ -359,6 +766,7 @@ export async function recordLayerAttentionGPU(
359
766
  { useF16Activations, matmulOutputDtype },
360
767
  usedFusedQKV, qTensor, kTensor, vTensor,
361
768
  ));
769
+ const mergedSessionRec = getRuntimeConfig()?.inference?.session;
362
770
 
363
771
  const attentionKernelRunners = {
364
772
  bdpa: async () => {
@@ -507,6 +915,14 @@ export async function recordLayerAttentionGPU(
507
915
  if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
508
916
  if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
509
917
  }
918
+ // Session precedence is runtime-over-manifest per config-style-guide
919
+ // §Category Rules. getRuntimeConfig() returns the merged session (manifest
920
+ // is the base layer, runtime fields win field-by-field via merge.js).
921
+ // Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
922
+ // when numTokens > 1 (prefill). Same flag semantics as the non-recorder
923
+ // path in ./run.js.
924
+ const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
925
+ const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
510
926
  return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
511
927
  seqLen: numTokens,
512
928
  kvLen: kvState.kvLenForAttention,
@@ -524,6 +940,8 @@ export async function recordLayerAttentionGPU(
524
940
  kvPageTable: kvState.kvPageTable,
525
941
  kvPageSize: kvState.kvPageSize,
526
942
  kernelPath,
943
+ useFlashPrefill: useFlashPrefillRec,
944
+ useOrtFlashPrefill: useOrtFlashPrefillRec,
527
945
  });
528
946
  },
529
947
  };
@@ -532,14 +950,44 @@ export async function recordLayerAttentionGPU(
532
950
  throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
533
951
  }
534
952
 
535
- attnOutput = await runAttentionKernel();
953
+ try {
954
+ attnOutput = await runAttentionKernel();
955
+ } finally {
956
+ if (decoderKVState?.ownedBuffers) {
957
+ for (const buffer of decoderKVState.ownedBuffers) {
958
+ recorder.trackTemporaryBuffer(buffer);
959
+ }
960
+ decoderKVState.ownedBuffers = null;
961
+ }
962
+ }
963
+ await runProbes('attn_core_out', attnOutput.buffer, {
964
+ layerIdx,
965
+ numTokens,
966
+ hiddenSize: numHeads * headDim,
967
+ probes: state.debugProbes,
968
+ recorder,
969
+ operatorDiagnostics: state.operatorDiagnostics,
970
+ dtype: attnOutput.dtype,
971
+ });
972
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
973
+ enqueueRecordedTensorHealth(
974
+ recorder,
975
+ `L${layerIdx}.attn_core_out_HEALTH`,
976
+ attnOutput,
977
+ attnOutput.dtype,
978
+ numTokens * numHeads * headDim
979
+ );
980
+ }
536
981
 
537
982
  attnForProjection = attnOutput;
538
983
  if (qGateTensor) {
984
+ // Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
985
+ // sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
986
+ const gateActivation = 'sigmoid';
539
987
  attnForProjection = await recordSiLU(recorder, attnOutput, {
540
988
  size: numTokens * numHeads * headDim,
541
989
  gate: qGateTensor,
542
- gateActivation: 'sigmoid',
990
+ gateActivation,
543
991
  inputActivation: 'identity',
544
992
  swigluLimit: null,
545
993
  });
@@ -584,7 +1032,16 @@ export async function recordLayerAttentionGPU(
584
1032
  hasLoRA: Boolean(loraO),
585
1033
  oProjIsF16: oProjDtype === 'f16',
586
1034
  });
587
-
1035
+ const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1036
+ phase,
1037
+ session: mergedSessionRec,
1038
+ hasResidual: Boolean(residualTensor),
1039
+ residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1040
+ inputDtype: oProjInput.dtype,
1041
+ outputDtype: oProjOutputDtype,
1042
+ oProjDtype,
1043
+ hasLoRA: Boolean(loraO),
1044
+ });
588
1045
  if (canUseFused && residualTensor) {
589
1046
  // FUSED PATH: o_proj matmul + residual add in one dispatch
590
1047
  output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
@@ -593,7 +1050,7 @@ export async function recordLayerAttentionGPU(
593
1050
  });
594
1051
  residualFused = true;
595
1052
  } else {
596
- // STANDARD PATH: o_proj matmul only
1053
+ // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
597
1054
  output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
598
1055
  transposeB: 'auto',
599
1056
  role: 'o_proj',
@@ -601,7 +1058,9 @@ export async function recordLayerAttentionGPU(
601
1058
  kernelPath,
602
1059
  outputDtype: oProjOutputDtype,
603
1060
  executionPolicies: state.executionPolicies ?? null,
1061
+ residualTensor: canUseWideTileResidual ? residualTensor : null,
604
1062
  });
1063
+ residualFused = canUseWideTileResidual;
605
1064
  }
606
1065
  // Release temporary buffer if we created it (original was not already on GPU)
607
1066
  if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
@@ -632,6 +1091,15 @@ export async function recordLayerAttentionGPU(
632
1091
  }
633
1092
 
634
1093
  finalOutput = output;
1094
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1095
+ enqueueRecordedTensorHealth(
1096
+ recorder,
1097
+ `L${layerIdx}.o_proj_HEALTH`,
1098
+ output,
1099
+ output.dtype,
1100
+ numTokens * hiddenSize
1101
+ );
1102
+ }
635
1103
 
636
1104
  const buffersToTrack = [];
637
1105
  if (output.buffer !== attnForProjection.buffer) {
@@ -653,13 +1121,19 @@ export async function recordLayerAttentionGPU(
653
1121
  // These buffers are used by recorded operations that haven't executed yet.
654
1122
  // Releasing them back to the pool would allow reuse before the encoder is submitted,
655
1123
  // causing data corruption (especially for small decode buffers).
656
- recorder.trackTemporaryBuffer(qTensor.buffer);
1124
+ if (qTensor) {
1125
+ recorder.trackTemporaryBuffer(qTensor.buffer);
1126
+ }
657
1127
  if (qGateTensor) {
658
1128
  recorder.trackTemporaryBuffer(qGateTensor.buffer);
659
1129
  }
660
1130
  if (!retainSharedKvBuffers) {
661
- recorder.trackTemporaryBuffer(kTensor.buffer);
662
- recorder.trackTemporaryBuffer(vTensor.buffer);
1131
+ if (kTensor) {
1132
+ recorder.trackTemporaryBuffer(kTensor.buffer);
1133
+ }
1134
+ if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
1135
+ recorder.trackTemporaryBuffer(vTensor.buffer);
1136
+ }
663
1137
  }
664
1138
  for (const buffer of buffersToTrack) {
665
1139
  recorder.trackTemporaryBuffer(buffer);
@@ -706,6 +1180,18 @@ export async function recordLayerAttentionGPU(
706
1180
  if (attentionInputTemp && attentionInput?.buffer) {
707
1181
  trackOnce(attentionInput.buffer);
708
1182
  }
1183
+ if (decoderKTemp?.buffer) {
1184
+ trackOnce(decoderKTemp.buffer);
1185
+ }
1186
+ if (decoderVTemp?.buffer) {
1187
+ trackOnce(decoderVTemp.buffer);
1188
+ }
1189
+ if (decoderKVState?.ownedBuffers) {
1190
+ for (const buffer of decoderKVState.ownedBuffers) {
1191
+ trackOnce(buffer);
1192
+ }
1193
+ decoderKVState.ownedBuffers = null;
1194
+ }
709
1195
  throw error;
710
1196
  }
711
1197
  }