doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -1,11 +1,14 @@
1
1
 
2
2
 
3
3
  import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { acquireBuffer } from '../../../../memory/buffer-pool.js';
4
+ import { getKernelCapabilities } from '../../../../gpu/device.js';
5
+ import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
5
6
  import {
6
7
  recordMatmul,
7
8
  recordRMSNorm,
8
9
  recordRoPE,
10
+ canUseRoPEQK,
11
+ recordRoPEQK,
9
12
  recordAttention,
10
13
  recordAttentionTiered,
11
14
  recordAttentionTieredQuant,
@@ -20,7 +23,7 @@ import {
20
23
  import { createTensor } from '../../../../gpu/tensor.js';
21
24
  import { applyLoRA } from '../lora-apply.js';
22
25
  import { getLoRAModule } from '../lora.js';
23
- import { log, trace } from '../../../../debug/index.js';
26
+ import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
24
27
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
25
28
  import {
26
29
  recordAttentionInputs,
@@ -29,9 +32,13 @@ import {
29
32
  projectAttentionQKV,
30
33
  applyAttentionQKNorm,
31
34
  applyAttentionValueNorm,
35
+ hasAttentionProjectionDiagnostics,
36
+ hasAttentionStageDiagnostics,
37
+ resolveAttentionQKNormState,
32
38
  } from './projections.js';
33
39
  import { prepareAttentionProjectionInput } from './output-projection.js';
34
40
  import { runProbes } from '../probes.js';
41
+ import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
35
42
 
36
43
  import { releaseOrTrack, shouldDebugLayer } from './types.js';
37
44
  import {
@@ -40,6 +47,7 @@ import {
40
47
  } from '../../../../config/kernel-path-loader.js';
41
48
  import {
42
49
  resolveKVCacheState,
50
+ createDiffusionGemmaDecoderKVState,
43
51
  buildAttentionDispatchParams,
44
52
  buildAttentionInputsData,
45
53
  } from './dispatch-params.js';
@@ -57,6 +65,93 @@ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js'
57
65
 
58
66
  const ATTENTION_DTYPE_LOGGED = new Set();
59
67
 
68
+ function isWideTileQ4KPhaseEnabled(session, phase) {
69
+ return phase === 'decode'
70
+ ? session?.useWideTileQ4KDecode === true
71
+ : session?.useWideTileQ4KPrefill === true;
72
+ }
73
+
74
+ function canUseQ4KWideTileResidualFusion(options = {}) {
75
+ return options.hasResidual === true
76
+ && options.oProjDtype === 'q4k'
77
+ && options.inputDtype === 'f32'
78
+ && options.outputDtype === 'f32'
79
+ && options.residualMatches === true
80
+ && options.hasLoRA !== true
81
+ && getKernelCapabilities().hasF16 === true
82
+ && getKernelCapabilities().hasSubgroups === true
83
+ && options.session?.useWideTileResidualFusion === true
84
+ && options.session?.retainQ4KMaterialization === true
85
+ && isWideTileQ4KPhaseEnabled(options.session, options.phase);
86
+ }
87
+
88
+ function shouldTraceRecordedHealth(layerIdx, debugFlags) {
89
+ const debugLayers = debugFlags?.debugLayers;
90
+ return isTraceEnabled('logits')
91
+ && Array.isArray(debugLayers)
92
+ && shouldDebugLayer(layerIdx, debugLayers);
93
+ }
94
+
95
+ function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
96
+ if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
97
+ return;
98
+ }
99
+ const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
100
+ recorder.enqueueCompletionTask(async () => {
101
+ const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
102
+ trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
103
+ });
104
+ }
105
+
106
+ function resolveDirectF16KVCacheWrite(options) {
107
+ const {
108
+ state,
109
+ layerIdx,
110
+ currentSeqLen,
111
+ numTokens,
112
+ numKVHeads,
113
+ headDim,
114
+ reusesSharedKV,
115
+ storeSharedKV,
116
+ diffusionGemmaDecoder,
117
+ valueNorm,
118
+ diagnosticsEnabled,
119
+ disableRoPE,
120
+ } = options;
121
+ if (
122
+ numTokens !== 1
123
+ || reusesSharedKV === true
124
+ || storeSharedKV === true
125
+ || diffusionGemmaDecoder === true
126
+ || valueNorm === true
127
+ || diagnosticsEnabled === true
128
+ || disableRoPE === true
129
+ ) {
130
+ return null;
131
+ }
132
+ const cache = state?.kvCache;
133
+ if (
134
+ !cache?.hasGPUCache?.()
135
+ || cache.kvDtype !== 'f16'
136
+ || cache.layout !== 'contiguous'
137
+ || cache.windowSize != null
138
+ || cache.layerSpecs != null
139
+ || typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
140
+ ) {
141
+ return null;
142
+ }
143
+ const gpuBuffers = cache.getGPUBuffers(layerIdx);
144
+ const layout = gpuBuffers?.layout ?? cache.layout;
145
+ if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
146
+ return null;
147
+ }
148
+ return {
149
+ keysBuffer: gpuBuffers.keysGPU,
150
+ valuesBuffer: gpuBuffers.valuesGPU,
151
+ dstOffset: currentSeqLen * numKVHeads * headDim,
152
+ };
153
+ }
154
+
60
155
  function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
61
156
  assertImplicitDtypeTransitionAllowed({
62
157
  executionPolicies: state?.executionPolicies ?? null,
@@ -104,6 +199,8 @@ export async function recordLayerAttentionGPU(
104
199
  multimodalBidirectionalSpan = null,
105
200
  sharedKVSourceLayerIdx = null,
106
201
  storeSharedKV = false,
202
+ diffusionGemmaDecoder = false,
203
+ precomputedInputNorm = null,
107
204
  } = config;
108
205
 
109
206
  const phase = isPrefill ? 'prefill' : 'decode';
@@ -125,6 +222,9 @@ export async function recordLayerAttentionGPU(
125
222
  const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
126
223
  const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
127
224
  const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
225
+ const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
226
+ ? 'step_precision'
227
+ : null;
128
228
  let attentionInput = input;
129
229
  let attentionInputTemp = false;
130
230
  let normed = attentionInput;
@@ -139,12 +239,16 @@ export async function recordLayerAttentionGPU(
139
239
  let oProjInputTemp = null;
140
240
  let retainSharedKvBuffers = false;
141
241
  let valueAliasesKey = false;
242
+ let decoderKVState = null;
243
+ let decoderKTemp = null;
244
+ let decoderVTemp = null;
142
245
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
143
246
  assertAttentionDtypeTransitionAllowed(
144
247
  state,
145
248
  input.dtype,
146
249
  attentionActivationDtype,
147
- 'The attention kernel selection would widen the input implicitly.'
250
+ 'The attention kernel selection would widen the input implicitly.',
251
+ attentionInputTransitionDeclaredBy
148
252
  );
149
253
  attentionInput = attentionActivationDtype === 'f16'
150
254
  ? await recordCastF32ToF16(recorder, input)
@@ -188,7 +292,25 @@ export async function recordLayerAttentionGPU(
188
292
  && attentionInput.dtype === 'f32';
189
293
 
190
294
  try {
191
- if (canFuseInputNormProjRec) {
295
+ if (precomputedInputNorm) {
296
+ if (
297
+ isPrefill
298
+ || numTokens !== 1
299
+ || skipInputNorm
300
+ || !layerWeights.inputNorm
301
+ || !getNormWeightBuffer
302
+ || precomputedInputNorm.dtype !== attentionInput.dtype
303
+ || precomputedInputNorm.dtype !== 'f32'
304
+ ) {
305
+ throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
306
+ }
307
+ normed = precomputedInputNorm;
308
+ await runProbes('post_input_norm', normed.buffer, {
309
+ layerIdx, numTokens, hiddenSize,
310
+ probes: state.debugProbes, recorder,
311
+ operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
312
+ });
313
+ } else if (canFuseInputNormProjRec) {
192
314
  fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
193
315
  fusedNormEpsRec = rmsNormEps;
194
316
  fusedNormOffsetRec = config.rmsNormWeightOffset === true;
@@ -200,6 +322,7 @@ export async function recordLayerAttentionGPU(
200
322
  batchSize: numTokens,
201
323
  hiddenSize,
202
324
  rmsNormWeightOffset: config.rmsNormWeightOffset,
325
+ label: 'input_norm',
203
326
  });
204
327
  if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
205
328
  await runProbes('post_input_norm', normed.buffer, {
@@ -254,7 +377,35 @@ export async function recordLayerAttentionGPU(
254
377
  }
255
378
  const reusesSharedKV = sharedKVEntry != null;
256
379
  retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
257
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
380
+ let qkNormApplied = false;
381
+ let ropeApplied = false;
382
+ const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
383
+ const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
384
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
385
+ const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
386
+ state,
387
+ ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
388
+ )
389
+ || shouldTraceRecordedHealth(layerIdx, debugFlags);
390
+ const runtimeSession = getRuntimeConfig()?.inference?.session;
391
+ const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
392
+ const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
393
+ const directF16KVCacheWrite = resolveDirectF16KVCacheWrite({
394
+ state,
395
+ layerIdx,
396
+ currentSeqLen,
397
+ numTokens,
398
+ numKVHeads,
399
+ headDim,
400
+ reusesSharedKV,
401
+ storeSharedKV,
402
+ diffusionGemmaDecoder,
403
+ valueNorm: config.valueNorm === true,
404
+ diagnosticsEnabled: qkNormRoPEDiagnostics,
405
+ disableRoPE,
406
+ });
407
+ let kvCacheWriteFused = false;
408
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
258
409
  recorder,
259
410
  normed,
260
411
  layerWeights,
@@ -282,16 +433,51 @@ export async function recordLayerAttentionGPU(
282
433
  fusedNormWeight: fusedNormWeightRec,
283
434
  fusedNormEps: fusedNormEpsRec,
284
435
  fusedNormOffset: fusedNormOffsetRec,
436
+ qkNormFusion: {
437
+ enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
438
+ getNormWeightBuffer,
439
+ rmsNormEps,
440
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
441
+ skipKNorm: qkNormState.skipKNorm,
442
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
443
+ projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
444
+ },
445
+ qkNormRoPEFusion: {
446
+ enabled: qkNormRoPEFusionFlag
447
+ && qkNormState.wantsQKNorm
448
+ && !disableRoPE
449
+ && !!state.ropeFreqsCos
450
+ && !!state.ropeFreqsSin,
451
+ getNormWeightBuffer,
452
+ rmsNormEps,
453
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
454
+ skipKNorm: qkNormState.skipKNorm,
455
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
456
+ projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
457
+ freqsCos: state.ropeFreqsCos,
458
+ freqsSin: state.ropeFreqsSin,
459
+ startPos: currentSeqLen,
460
+ rotaryDim: config.ropeRotaryDim,
461
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
462
+ interleaved: config.ropeInterleaved,
463
+ reusesSharedKV,
464
+ f16KVCacheWrite: directF16KVCacheWrite,
465
+ },
285
466
  }));
467
+ if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
468
+ throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
469
+ }
286
470
  // Deferred release of the norm weight buffer for fused path.
287
471
  if (fusedNormWeightRec && fusedNormOwnedRec) {
288
472
  releaseOrTrack(recorder, fusedNormWeightRec);
289
473
  }
290
- await runProbes('q_proj', qTensor.buffer, {
291
- layerIdx, numTokens, hiddenSize: numHeads * headDim,
292
- probes: state.debugProbes, recorder,
293
- operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
294
- });
474
+ if (qTensor) {
475
+ await runProbes('q_proj', qTensor.buffer, {
476
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
477
+ probes: state.debugProbes, recorder,
478
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
479
+ });
480
+ }
295
481
  if (kTensor) {
296
482
  await runProbes('k_proj', kTensor.buffer, {
297
483
  layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
@@ -306,43 +492,83 @@ export async function recordLayerAttentionGPU(
306
492
  operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
307
493
  });
308
494
  }
495
+ if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
496
+ enqueueRecordedTensorHealth(
497
+ recorder,
498
+ `L${layerIdx}.q_proj_HEALTH`,
499
+ qTensor,
500
+ qTensor.dtype,
501
+ numTokens * numHeads * headDim
502
+ );
503
+ enqueueRecordedTensorHealth(
504
+ recorder,
505
+ `L${layerIdx}.k_proj_HEALTH`,
506
+ kTensor,
507
+ kTensor?.dtype ?? null,
508
+ numTokens * numKVHeads * headDim
509
+ );
510
+ enqueueRecordedTensorHealth(
511
+ recorder,
512
+ `L${layerIdx}.v_proj_HEALTH`,
513
+ vTensor,
514
+ vTensor?.dtype ?? null,
515
+ numTokens * numKVHeads * headDim
516
+ );
517
+ }
309
518
 
310
519
  // Optional per-head Q/K normalization.
311
520
  // Some models use RMSNorm with (1+weight) offset formula, controlled by rmsNormWeightOffset.
312
- const wantsQKNorm = config.queryKeyNorm === true;
313
- if (wantsQKNorm && layerIdx === 0 && (!layerWeights.qNorm || !layerWeights.kNorm)) {
314
- log.warn('Attention', `Q/K norm requested but weights missing (hasQ=${!!layerWeights.qNorm}, hasK=${!!layerWeights.kNorm}); skipping QK norm.`);
521
+ if (qkNormState.wantsQKNorm && !qkNormApplied) {
522
+ ({ qTensor, kTensor } = await applyAttentionQKNorm({
523
+ recorder,
524
+ qTensor,
525
+ kTensor,
526
+ layerWeights,
527
+ getNormWeightBuffer,
528
+ rmsNormEps,
529
+ numTokens,
530
+ numHeads,
531
+ numKVHeads,
532
+ headDim,
533
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
534
+ releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
535
+ skipKNorm: qkNormState.skipKNorm,
536
+ retainKInput: valueAliasesKey,
537
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
538
+ }));
315
539
  }
316
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
317
- recorder,
318
- qTensor,
319
- kTensor,
320
- layerWeights,
321
- getNormWeightBuffer,
322
- rmsNormEps,
323
- numTokens,
324
- numHeads,
325
- numKVHeads,
326
- headDim,
327
- rmsNormWeightOffset: config.rmsNormWeightOffset,
328
- releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
329
- skipKNorm: reusesSharedKV,
330
- retainKInput: valueAliasesKey,
331
- }));
332
- await runProbes('q_norm', qTensor.buffer, {
333
- layerIdx, numTokens, hiddenSize: numHeads * headDim,
334
- probes: state.debugProbes, recorder,
335
- operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
336
- });
337
- if (kTensor) {
338
- await runProbes('k_norm', kTensor.buffer, {
339
- layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
540
+ if (qkNormState.wantsQKNorm) {
541
+ await runProbes('q_norm', qTensor.buffer, {
542
+ layerIdx, numTokens, hiddenSize: numHeads * headDim,
340
543
  probes: state.debugProbes, recorder,
341
- operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
544
+ operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
342
545
  });
546
+ if (kTensor) {
547
+ await runProbes('k_norm', kTensor.buffer, {
548
+ layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
549
+ probes: state.debugProbes, recorder,
550
+ operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
551
+ });
552
+ }
553
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
554
+ enqueueRecordedTensorHealth(
555
+ recorder,
556
+ `L${layerIdx}.q_norm_HEALTH`,
557
+ qTensor,
558
+ qTensor.dtype,
559
+ numTokens * numHeads * headDim
560
+ );
561
+ enqueueRecordedTensorHealth(
562
+ recorder,
563
+ `L${layerIdx}.k_norm_HEALTH`,
564
+ kTensor,
565
+ kTensor?.dtype ?? null,
566
+ numTokens * numKVHeads * headDim
567
+ );
568
+ }
343
569
  }
344
570
 
345
- if (config.valueNorm === true && !reusesSharedKV) {
571
+ if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
346
572
  const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
347
573
  vTensor = await applyAttentionValueNorm({
348
574
  recorder,
@@ -368,28 +594,50 @@ export async function recordLayerAttentionGPU(
368
594
  if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
369
595
 
370
596
  // 3. RoPE (modifies tensor in-place)
371
- if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
372
- await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
373
- numHeads,
597
+ if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
598
+ const ropeOptions = {
374
599
  headDim,
375
600
  rotaryDim: config.ropeRotaryDim,
376
601
  pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
377
602
  interleaved: config.ropeInterleaved,
378
603
  startPos: currentSeqLen,
379
604
  executionPolicies: state.executionPolicies ?? null,
380
- });
381
- if (!reusesSharedKV) {
382
- await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
383
- numHeads: numKVHeads,
384
- headDim,
385
- rotaryDim: config.ropeRotaryDim,
386
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
387
- interleaved: config.ropeInterleaved,
388
- startPos: currentSeqLen,
389
- executionPolicies: state.executionPolicies ?? null,
605
+ };
606
+ if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
607
+ await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
608
+ ...ropeOptions,
609
+ numQHeads: numHeads,
610
+ numKVHeads,
611
+ });
612
+ } else {
613
+ await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
614
+ ...ropeOptions,
615
+ numHeads,
390
616
  });
617
+ if (!reusesSharedKV) {
618
+ await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
619
+ ...ropeOptions,
620
+ numHeads: numKVHeads,
621
+ });
622
+ }
391
623
  }
392
624
  }
625
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
626
+ enqueueRecordedTensorHealth(
627
+ recorder,
628
+ `L${layerIdx}.q_rope_HEALTH`,
629
+ qTensor,
630
+ qTensor.dtype,
631
+ numTokens * numHeads * headDim
632
+ );
633
+ enqueueRecordedTensorHealth(
634
+ recorder,
635
+ `L${layerIdx}.k_rope_HEALTH`,
636
+ kTensor,
637
+ kTensor?.dtype ?? null,
638
+ numTokens * numKVHeads * headDim
639
+ );
640
+ }
393
641
 
394
642
  if (storeSharedKV && state.sharedAttentionState) {
395
643
  state.sharedAttentionState.set(layerIdx, {
@@ -401,10 +649,10 @@ export async function recordLayerAttentionGPU(
401
649
  }
402
650
 
403
651
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
404
- if (state.kvCache?.hasGPUCache?.()) {
405
- // Use recordUpdateFromGPU to record copy operations to the recorder's encoder
406
- // This ensures K/V buffers are populated before copying (all ops submitted together)
407
- if (state.kvCache.kvDtype === 'f16') {
652
+ if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
653
+ if (kvCacheWriteFused) {
654
+ state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
655
+ } else if (state.kvCache.kvDtype === 'f16') {
408
656
  const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
409
657
  if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
410
658
  assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
@@ -412,24 +660,95 @@ export async function recordLayerAttentionGPU(
412
660
  if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
413
661
  assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
414
662
  }
415
- const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
416
- const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
663
+ const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
664
+ && vTensor.dtype === 'f32'
665
+ && typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
666
+ if (canWriteDirectF32ToF16) {
667
+ await state.kvCache.recordUpdateF32ToF16FromGPU(
668
+ recorder,
669
+ layerIdx,
670
+ kTensor.buffer,
671
+ vTensor.buffer,
672
+ currentSeqLen,
673
+ numTokens,
674
+ tokenIds
675
+ );
676
+ } else {
677
+ const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
678
+ const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
417
679
 
418
- await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
680
+ await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
419
681
 
420
- if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
421
- if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
682
+ if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
683
+ if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
684
+ }
422
685
  } else {
423
686
  await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
424
687
  }
425
688
  }
426
689
 
427
690
  // Resolve KV cache state and build dispatch parameters (shared with run.js)
428
- const kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
691
+ let kvState;
692
+ if (diffusionGemmaDecoder) {
693
+ const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
694
+ let decoderKTensor = kTensor;
695
+ let decoderVTensor = vTensor;
696
+ if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
697
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
698
+ if (!hasExplicitF16KvContract) {
699
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
700
+ }
701
+ decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
702
+ decoderKTemp = decoderKTensor;
703
+ } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
704
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
705
+ decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
706
+ decoderKTemp = decoderKTensor;
707
+ }
708
+ if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
709
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
710
+ if (!hasExplicitF16KvContract) {
711
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
712
+ }
713
+ decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
714
+ decoderVTemp = decoderVTensor;
715
+ } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
716
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
717
+ decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
718
+ decoderVTemp = decoderVTensor;
719
+ }
720
+ kvState = await createDiffusionGemmaDecoderKVState({
721
+ state,
722
+ layerIdx,
723
+ kTensor: decoderKTensor,
724
+ vTensor: decoderVTensor,
725
+ currentSeqLen,
726
+ numTokens,
727
+ numKVHeads,
728
+ headDim,
729
+ layerType,
730
+ slidingWindow,
731
+ kvDtype: decoderKVDtype,
732
+ recorder,
733
+ });
734
+ if (decoderKTemp) {
735
+ recorder.trackTemporaryBuffer(decoderKTemp.buffer);
736
+ decoderKTemp = null;
737
+ }
738
+ if (decoderVTemp) {
739
+ recorder.trackTemporaryBuffer(decoderVTemp.buffer);
740
+ decoderVTemp = null;
741
+ }
742
+ decoderKVState = kvState;
743
+ } else {
744
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
745
+ }
429
746
  const dispatchConfig = {
430
747
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
431
- slidingWindow, layerType, layerTypes: config.layerTypes,
432
- queryPreAttnScalar, causalAttention: config.causalAttention,
748
+ slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
749
+ layerType, layerTypes: config.layerTypes,
750
+ queryPreAttnScalar,
751
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
433
752
  activationDtype: attentionActivationDtype,
434
753
  kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
435
754
  };
@@ -447,6 +766,7 @@ export async function recordLayerAttentionGPU(
447
766
  { useF16Activations, matmulOutputDtype },
448
767
  usedFusedQKV, qTensor, kTensor, vTensor,
449
768
  ));
769
+ const mergedSessionRec = getRuntimeConfig()?.inference?.session;
450
770
 
451
771
  const attentionKernelRunners = {
452
772
  bdpa: async () => {
@@ -601,9 +921,8 @@ export async function recordLayerAttentionGPU(
601
921
  // Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
602
922
  // when numTokens > 1 (prefill). Same flag semantics as the non-recorder
603
923
  // path in ./run.js.
604
- const mergedSessionRec = getRuntimeConfig()?.inference?.session;
605
- const useFlashPrefillRec = mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
606
- const useOrtFlashPrefillRec = mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
924
+ const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
925
+ const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
607
926
  return recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
608
927
  seqLen: numTokens,
609
928
  kvLen: kvState.kvLenForAttention,
@@ -631,7 +950,16 @@ export async function recordLayerAttentionGPU(
631
950
  throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
632
951
  }
633
952
 
634
- attnOutput = await runAttentionKernel();
953
+ try {
954
+ attnOutput = await runAttentionKernel();
955
+ } finally {
956
+ if (decoderKVState?.ownedBuffers) {
957
+ for (const buffer of decoderKVState.ownedBuffers) {
958
+ recorder.trackTemporaryBuffer(buffer);
959
+ }
960
+ decoderKVState.ownedBuffers = null;
961
+ }
962
+ }
635
963
  await runProbes('attn_core_out', attnOutput.buffer, {
636
964
  layerIdx,
637
965
  numTokens,
@@ -641,6 +969,15 @@ export async function recordLayerAttentionGPU(
641
969
  operatorDiagnostics: state.operatorDiagnostics,
642
970
  dtype: attnOutput.dtype,
643
971
  });
972
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
973
+ enqueueRecordedTensorHealth(
974
+ recorder,
975
+ `L${layerIdx}.attn_core_out_HEALTH`,
976
+ attnOutput,
977
+ attnOutput.dtype,
978
+ numTokens * numHeads * headDim
979
+ );
980
+ }
644
981
 
645
982
  attnForProjection = attnOutput;
646
983
  if (qGateTensor) {
@@ -695,7 +1032,16 @@ export async function recordLayerAttentionGPU(
695
1032
  hasLoRA: Boolean(loraO),
696
1033
  oProjIsF16: oProjDtype === 'f16',
697
1034
  });
698
-
1035
+ const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1036
+ phase,
1037
+ session: mergedSessionRec,
1038
+ hasResidual: Boolean(residualTensor),
1039
+ residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1040
+ inputDtype: oProjInput.dtype,
1041
+ outputDtype: oProjOutputDtype,
1042
+ oProjDtype,
1043
+ hasLoRA: Boolean(loraO),
1044
+ });
699
1045
  if (canUseFused && residualTensor) {
700
1046
  // FUSED PATH: o_proj matmul + residual add in one dispatch
701
1047
  output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
@@ -704,7 +1050,7 @@ export async function recordLayerAttentionGPU(
704
1050
  });
705
1051
  residualFused = true;
706
1052
  } else {
707
- // STANDARD PATH: o_proj matmul only
1053
+ // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
708
1054
  output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
709
1055
  transposeB: 'auto',
710
1056
  role: 'o_proj',
@@ -712,7 +1058,9 @@ export async function recordLayerAttentionGPU(
712
1058
  kernelPath,
713
1059
  outputDtype: oProjOutputDtype,
714
1060
  executionPolicies: state.executionPolicies ?? null,
1061
+ residualTensor: canUseWideTileResidual ? residualTensor : null,
715
1062
  });
1063
+ residualFused = canUseWideTileResidual;
716
1064
  }
717
1065
  // Release temporary buffer if we created it (original was not already on GPU)
718
1066
  if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
@@ -743,6 +1091,15 @@ export async function recordLayerAttentionGPU(
743
1091
  }
744
1092
 
745
1093
  finalOutput = output;
1094
+ if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
1095
+ enqueueRecordedTensorHealth(
1096
+ recorder,
1097
+ `L${layerIdx}.o_proj_HEALTH`,
1098
+ output,
1099
+ output.dtype,
1100
+ numTokens * hiddenSize
1101
+ );
1102
+ }
746
1103
 
747
1104
  const buffersToTrack = [];
748
1105
  if (output.buffer !== attnForProjection.buffer) {
@@ -764,13 +1121,17 @@ export async function recordLayerAttentionGPU(
764
1121
  // These buffers are used by recorded operations that haven't executed yet.
765
1122
  // Releasing them back to the pool would allow reuse before the encoder is submitted,
766
1123
  // causing data corruption (especially for small decode buffers).
767
- recorder.trackTemporaryBuffer(qTensor.buffer);
1124
+ if (qTensor) {
1125
+ recorder.trackTemporaryBuffer(qTensor.buffer);
1126
+ }
768
1127
  if (qGateTensor) {
769
1128
  recorder.trackTemporaryBuffer(qGateTensor.buffer);
770
1129
  }
771
1130
  if (!retainSharedKvBuffers) {
772
- recorder.trackTemporaryBuffer(kTensor.buffer);
773
- if (vTensor.buffer !== kTensor.buffer) {
1131
+ if (kTensor) {
1132
+ recorder.trackTemporaryBuffer(kTensor.buffer);
1133
+ }
1134
+ if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
774
1135
  recorder.trackTemporaryBuffer(vTensor.buffer);
775
1136
  }
776
1137
  }
@@ -819,6 +1180,18 @@ export async function recordLayerAttentionGPU(
819
1180
  if (attentionInputTemp && attentionInput?.buffer) {
820
1181
  trackOnce(attentionInput.buffer);
821
1182
  }
1183
+ if (decoderKTemp?.buffer) {
1184
+ trackOnce(decoderKTemp.buffer);
1185
+ }
1186
+ if (decoderVTemp?.buffer) {
1187
+ trackOnce(decoderVTemp.buffer);
1188
+ }
1189
+ if (decoderKVState?.ownedBuffers) {
1190
+ for (const buffer of decoderKVState.ownedBuffers) {
1191
+ trackOnce(buffer);
1192
+ }
1193
+ decoderKVState.ownedBuffers = null;
1194
+ }
822
1195
  throw error;
823
1196
  }
824
1197
  }