doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -1,13 +1,15 @@
1
1
 
2
2
 
3
3
  import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { getDevice } from '../../../../gpu/device.js';
4
+ import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
5
5
  import { getRuntimeConfig } from '../../../../config/runtime.js';
6
6
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
7
7
  import {
8
8
  runMatmul,
9
9
  runRMSNorm,
10
10
  runRoPE,
11
+ canUseRoPEQK,
12
+ runRoPEQK,
11
13
  runAttention,
12
14
  runAttentionBDPA,
13
15
  runAttentionTieredQuant,
@@ -34,6 +36,9 @@ import {
34
36
  projectAttentionQKV,
35
37
  applyAttentionQKNorm,
36
38
  applyAttentionValueNorm,
39
+ hasAttentionProjectionDiagnostics,
40
+ hasAttentionStageDiagnostics,
41
+ resolveAttentionQKNormState,
37
42
  } from './projections.js';
38
43
  import { prepareAttentionProjectionInput } from './output-projection.js';
39
44
 
@@ -47,6 +52,7 @@ import {
47
52
  } from '../../../../config/kernel-path-loader.js';
48
53
  import {
49
54
  resolveKVCacheState,
55
+ createDiffusionGemmaDecoderKVState,
50
56
  buildAttentionDispatchParams,
51
57
  buildAttentionInputsData,
52
58
  } from './dispatch-params.js';
@@ -63,6 +69,26 @@ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js'
63
69
 
64
70
  const ATTENTION_DTYPE_LOGGED = new Set();
65
71
 
72
+ function isWideTileQ4KPhaseEnabled(session, phase) {
73
+ return phase === 'decode'
74
+ ? session?.useWideTileQ4KDecode === true
75
+ : session?.useWideTileQ4KPrefill === true;
76
+ }
77
+
78
+ function canUseQ4KWideTileResidualFusion(options = {}) {
79
+ return options.hasResidual === true
80
+ && options.oProjDtype === 'q4k'
81
+ && options.inputDtype === 'f32'
82
+ && options.outputDtype === 'f32'
83
+ && options.residualMatches === true
84
+ && options.hasLoRA !== true
85
+ && getKernelCapabilities().hasF16 === true
86
+ && getKernelCapabilities().hasSubgroups === true
87
+ && options.session?.useWideTileResidualFusion === true
88
+ && options.session?.retainQ4KMaterialization === true
89
+ && isWideTileQ4KPhaseEnabled(options.session, options.phase);
90
+ }
91
+
66
92
  function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
67
93
  assertImplicitDtypeTransitionAllowed({
68
94
  executionPolicies: state?.executionPolicies ?? null,
@@ -108,6 +134,8 @@ export async function runLayerAttentionGPU(
108
134
  multimodalBidirectionalSpan = null,
109
135
  sharedKVSourceLayerIdx = null,
110
136
  storeSharedKV = false,
137
+ diffusionGemmaDecoder = false,
138
+ precomputedInputNorm = null,
111
139
  } = config;
112
140
 
113
141
  const device = getDevice();
@@ -130,6 +158,9 @@ export async function runLayerAttentionGPU(
130
158
  const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
131
159
  const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
132
160
  const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
161
+ const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
162
+ ? 'step_precision'
163
+ : null;
133
164
  let attentionInput = input;
134
165
  let attentionInputTemp = false;
135
166
  let normed = attentionInput;
@@ -143,12 +174,16 @@ export async function runLayerAttentionGPU(
143
174
  let finalOutput = null;
144
175
  let oProjInputTemp = null;
145
176
  let valueAliasesKey = false;
177
+ let decoderKVState = null;
178
+ let decoderKTemp = null;
179
+ let decoderVTemp = null;
146
180
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
147
181
  assertAttentionDtypeTransitionAllowed(
148
182
  state,
149
183
  input.dtype,
150
184
  attentionActivationDtype,
151
- 'The attention kernel selection would widen the input implicitly.'
185
+ 'The attention kernel selection would widen the input implicitly.',
186
+ attentionInputTransitionDeclaredBy
152
187
  );
153
188
  attentionInput = attentionActivationDtype === 'f16'
154
189
  ? await castF32ToF16(input)
@@ -216,7 +251,28 @@ export async function runLayerAttentionGPU(
216
251
  && attentionInput.dtype === 'f32';
217
252
 
218
253
  try {
219
- if (canFuseInputNormIntoProjections) {
254
+ if (precomputedInputNorm) {
255
+ if (
256
+ isPrefill
257
+ || numTokens !== 1
258
+ || skipInputNorm
259
+ || !layerWeights.inputNorm
260
+ || !getNormWeightBuffer
261
+ || precomputedInputNorm.dtype !== attentionInput.dtype
262
+ || precomputedInputNorm.dtype !== 'f32'
263
+ ) {
264
+ throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
265
+ }
266
+ normed = precomputedInputNorm;
267
+ await runProbes('post_input_norm', normed.buffer, {
268
+ layerIdx,
269
+ numTokens,
270
+ hiddenSize,
271
+ probes: state.debugProbes,
272
+ operatorDiagnostics: state.operatorDiagnostics,
273
+ dtype: normed.dtype,
274
+ });
275
+ } else if (canFuseInputNormIntoProjections) {
220
276
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
221
277
  fusedNormWeight = normWeightBuf;
222
278
  fusedNormEps = rmsNormEps;
@@ -305,7 +361,24 @@ export async function runLayerAttentionGPU(
305
361
  }),
306
362
  });
307
363
  let usedFusedQKV = false;
308
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey } = await projectAttentionQKV({
364
+ let qkNormApplied = false;
365
+ let ropeApplied = false;
366
+ const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
367
+ const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
368
+ || kernelTrace.enabled
369
+ || isKernelDebugEnabled(layerIdx)
370
+ || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
371
+ const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
372
+ state,
373
+ ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
374
+ )
375
+ || kernelTrace.enabled
376
+ || isKernelDebugEnabled(layerIdx)
377
+ || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
378
+ const runtimeSession = getRuntimeConfig()?.inference?.session;
379
+ const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
380
+ const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
381
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
309
382
  recorder: null,
310
383
  normed,
311
384
  layerWeights,
@@ -333,6 +406,35 @@ export async function runLayerAttentionGPU(
333
406
  fusedNormWeight,
334
407
  fusedNormEps,
335
408
  fusedNormOffset,
409
+ qkNormFusion: {
410
+ enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
411
+ getNormWeightBuffer,
412
+ rmsNormEps,
413
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
414
+ skipKNorm: qkNormState.skipKNorm,
415
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
416
+ projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
417
+ },
418
+ qkNormRoPEFusion: {
419
+ enabled: qkNormRoPEFusionFlag
420
+ && qkNormState.wantsQKNorm
421
+ && !disableRoPE
422
+ && !!state.ropeFreqsCos
423
+ && !!state.ropeFreqsSin,
424
+ getNormWeightBuffer,
425
+ rmsNormEps,
426
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
427
+ skipKNorm: qkNormState.skipKNorm,
428
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
429
+ projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
430
+ freqsCos: state.ropeFreqsCos,
431
+ freqsSin: state.ropeFreqsSin,
432
+ startPos: currentSeqLen,
433
+ rotaryDim: config.ropeRotaryDim,
434
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
435
+ interleaved: config.ropeInterleaved,
436
+ reusesSharedKV,
437
+ },
336
438
  }));
337
439
  // Release the in-flight norm weight buffer used by the fused projections.
338
440
  if (fusedNormWeight && fusedNormWeightOwned) {
@@ -406,17 +508,19 @@ export async function runLayerAttentionGPU(
406
508
  }
407
509
 
408
510
  // Optional per-head Q/K normalization
409
- const wantsQKNorm = config.queryKeyNorm === true;
410
- const hasQNorm = !!layerWeights.qNorm;
411
- const hasKNorm = !!layerWeights.kNorm;
412
511
  if (isKernelDebugEnabled(layerIdx)) {
413
- logKernelStep('qk_norm', { layerIdx, label: `hasQ=${hasQNorm} hasK=${hasKNorm} wants=${wantsQKNorm}` });
414
- }
415
- if (wantsQKNorm && layerIdx === 0 && (!hasQNorm || !hasKNorm)) {
416
- log.warn('Attention', `Q/K norm requested but weights missing (hasQ=${hasQNorm}, hasK=${hasKNorm}); skipping QK norm.`);
512
+ logKernelStep(
513
+ 'qk_norm',
514
+ {
515
+ layerIdx,
516
+ label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
517
+ `wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
518
+ }
519
+ );
417
520
  }
418
521
 
419
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
522
+ if (qkNormState.wantsQKNorm && !qkNormApplied) {
523
+ ({ qTensor, kTensor } = await applyAttentionQKNorm({
420
524
  recorder: null,
421
525
  qTensor,
422
526
  kTensor,
@@ -429,7 +533,7 @@ export async function runLayerAttentionGPU(
429
533
  headDim,
430
534
  rmsNormWeightOffset: config.rmsNormWeightOffset,
431
535
  releaseTemporary: (buffer) => releaseBuffer(buffer),
432
- skipKNorm: reusesSharedKV,
536
+ skipKNorm: qkNormState.skipKNorm,
433
537
  onQNormApplied: isKernelDebugEnabled(layerIdx)
434
538
  ? async (tensor) => {
435
539
  await dumpTokenVector(tensor.buffer, 'Q_norm', {
@@ -449,9 +553,11 @@ export async function runLayerAttentionGPU(
449
553
  dtype: tensor.dtype,
450
554
  });
451
555
  }
452
- : null,
556
+ : null,
453
557
  retainKInput: valueAliasesKey,
454
- }));
558
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
559
+ }));
560
+ }
455
561
 
456
562
  if (config.valueNorm === true && !reusesSharedKV) {
457
563
  const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
@@ -478,23 +584,25 @@ export async function runLayerAttentionGPU(
478
584
  });
479
585
  }
480
586
 
481
- await runProbes('q_norm', qTensor.buffer, {
482
- layerIdx,
483
- numTokens,
484
- hiddenSize: numHeads * headDim,
485
- probes: state.debugProbes,
486
- operatorDiagnostics: state.operatorDiagnostics,
487
- dtype: qTensor.dtype,
488
- });
489
- if (!reusesSharedKV) {
490
- await runProbes('k_norm', kTensor.buffer, {
587
+ if (qkNormState.wantsQKNorm) {
588
+ await runProbes('q_norm', qTensor.buffer, {
491
589
  layerIdx,
492
590
  numTokens,
493
- hiddenSize: numKVHeads * headDim,
591
+ hiddenSize: numHeads * headDim,
494
592
  probes: state.debugProbes,
495
593
  operatorDiagnostics: state.operatorDiagnostics,
496
- dtype: kTensor.dtype,
594
+ dtype: qTensor.dtype,
497
595
  });
596
+ if (!reusesSharedKV) {
597
+ await runProbes('k_norm', kTensor.buffer, {
598
+ layerIdx,
599
+ numTokens,
600
+ hiddenSize: numKVHeads * headDim,
601
+ probes: state.debugProbes,
602
+ operatorDiagnostics: state.operatorDiagnostics,
603
+ dtype: kTensor.dtype,
604
+ });
605
+ }
498
606
  }
499
607
 
500
608
  if (normed !== attentionInput) releaseBuffer(normed.buffer);
@@ -502,26 +610,32 @@ export async function runLayerAttentionGPU(
502
610
 
503
611
  // 3. RoPE (modifies tensor in-place)
504
612
 
505
- if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
506
- await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
507
- numHeads,
613
+ if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
614
+ const ropeOptions = {
508
615
  headDim,
509
616
  rotaryDim: config.ropeRotaryDim,
510
617
  pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
511
618
  interleaved: config.ropeInterleaved,
512
619
  startPos: currentSeqLen,
513
620
  executionPolicies: state.executionPolicies ?? null,
514
- });
515
- if (!reusesSharedKV) {
516
- await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
517
- numHeads: numKVHeads,
518
- headDim,
519
- rotaryDim: config.ropeRotaryDim,
520
- pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
521
- interleaved: config.ropeInterleaved,
522
- startPos: currentSeqLen,
523
- executionPolicies: state.executionPolicies ?? null,
621
+ };
622
+ if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
623
+ await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
624
+ ...ropeOptions,
625
+ numQHeads: numHeads,
626
+ numKVHeads,
524
627
  });
628
+ } else {
629
+ await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
630
+ ...ropeOptions,
631
+ numHeads,
632
+ });
633
+ if (!reusesSharedKV) {
634
+ await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
635
+ ...ropeOptions,
636
+ numHeads: numKVHeads,
637
+ });
638
+ }
525
639
  }
526
640
 
527
641
  // Trace RoPE outputs
@@ -582,7 +696,7 @@ export async function runLayerAttentionGPU(
582
696
 
583
697
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
584
698
 
585
- if (state.kvCache?.hasGPUCache?.()) {
699
+ if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
586
700
  if (state.kvCache.kvDtype === 'f16') {
587
701
  const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
588
702
  if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
@@ -610,11 +724,66 @@ export async function runLayerAttentionGPU(
610
724
  }
611
725
 
612
726
  // Resolve KV cache state and build dispatch parameters (shared with record.js)
613
- const kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
727
+ let kvState;
728
+ if (diffusionGemmaDecoder) {
729
+ const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
730
+ let decoderKTensor = kTensor;
731
+ let decoderVTensor = vTensor;
732
+ if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
733
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
734
+ if (!hasExplicitF16KvContract) {
735
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
736
+ }
737
+ decoderKTensor = await castF32ToF16(decoderKTensor);
738
+ decoderKTemp = decoderKTensor;
739
+ } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
740
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
741
+ decoderKTensor = await castF16ToF32(decoderKTensor);
742
+ decoderKTemp = decoderKTensor;
743
+ }
744
+ if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
745
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
746
+ if (!hasExplicitF16KvContract) {
747
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
748
+ }
749
+ decoderVTensor = await castF32ToF16(decoderVTensor);
750
+ decoderVTemp = decoderVTensor;
751
+ } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
752
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
753
+ decoderVTensor = await castF16ToF32(decoderVTensor);
754
+ decoderVTemp = decoderVTensor;
755
+ }
756
+ kvState = await createDiffusionGemmaDecoderKVState({
757
+ state,
758
+ layerIdx,
759
+ kTensor: decoderKTensor,
760
+ vTensor: decoderVTensor,
761
+ currentSeqLen,
762
+ numTokens,
763
+ numKVHeads,
764
+ headDim,
765
+ layerType,
766
+ slidingWindow,
767
+ kvDtype: decoderKVDtype,
768
+ });
769
+ if (decoderKTemp) {
770
+ releaseBuffer(decoderKTemp.buffer);
771
+ decoderKTemp = null;
772
+ }
773
+ if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
774
+ releaseBuffer(decoderVTemp.buffer);
775
+ }
776
+ decoderVTemp = null;
777
+ } else {
778
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
779
+ }
614
780
  const dispatchConfig = {
615
781
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
616
- slidingWindow, layerType, layerTypes: config.layerTypes,
617
- queryPreAttnScalar, causalAttention: config.causalAttention,
782
+ slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
783
+ layerType,
784
+ layerTypes: config.layerTypes,
785
+ queryPreAttnScalar,
786
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
618
787
  activationDtype: attentionActivationDtype,
619
788
  kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
620
789
  };
@@ -823,8 +992,8 @@ export async function runLayerAttentionGPU(
823
992
  // see comment above at the rmsNorm fusion check). The kernel itself
824
993
  // enforces remaining preconditions (head_dim=256, etc.).
825
994
  const mergedSession = getRuntimeConfig()?.inference?.session;
826
- const useFlashPrefill = mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
827
- const useOrtFlashPrefill = mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
995
+ const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
996
+ const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
828
997
  const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
829
998
  seqLen: numTokens,
830
999
  kvLen: kvState.kvLenForAttention,
@@ -858,7 +1027,16 @@ export async function runLayerAttentionGPU(
858
1027
  throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
859
1028
  }
860
1029
 
861
- attnOutput = await runAttentionKernel();
1030
+ try {
1031
+ attnOutput = await runAttentionKernel();
1032
+ } finally {
1033
+ if (decoderKVState?.ownedBuffers) {
1034
+ for (const buffer of decoderKVState.ownedBuffers) {
1035
+ releaseBuffer(buffer);
1036
+ }
1037
+ decoderKVState.ownedBuffers = null;
1038
+ }
1039
+ }
862
1040
  await runProbes('attn_core_out', attnOutput.buffer, {
863
1041
  layerIdx,
864
1042
  numTokens,
@@ -947,6 +1125,7 @@ export async function runLayerAttentionGPU(
947
1125
  // Use fused o_proj + residual for decode when possible
948
1126
  // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
949
1127
  const oProjDtype = getWeightDtype(oProjBuf);
1128
+ const mergedSession = getRuntimeConfig()?.inference?.session;
950
1129
  const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
951
1130
  allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
952
1131
  hasResidual: Boolean(residualTensor),
@@ -956,7 +1135,16 @@ export async function runLayerAttentionGPU(
956
1135
  hasLoRA: Boolean(loraO),
957
1136
  oProjIsF16: oProjDtype === 'f16',
958
1137
  }); // GEMV kernel expects f16 weights
959
-
1138
+ const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1139
+ phase,
1140
+ session: mergedSession,
1141
+ hasResidual: Boolean(residualTensor),
1142
+ residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1143
+ inputDtype: oProjInput.dtype,
1144
+ outputDtype: oProjOutputDtype,
1145
+ oProjDtype,
1146
+ hasLoRA: Boolean(loraO),
1147
+ });
960
1148
  if (canUseFused && residualTensor) {
961
1149
  // FUSED PATH: o_proj matmul + residual add in one dispatch
962
1150
  output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
@@ -969,7 +1157,7 @@ export async function runLayerAttentionGPU(
969
1157
  trace.attn(layerIdx, `Using fused o_proj+residual path`);
970
1158
  }
971
1159
  } else {
972
- // STANDARD PATH: o_proj matmul only (residual will be added by layer.ts)
1160
+ // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
973
1161
  output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
974
1162
  transposeB: 'auto',
975
1163
  role: 'o_proj',
@@ -977,7 +1165,9 @@ export async function runLayerAttentionGPU(
977
1165
  kernelPath,
978
1166
  outputDtype: oProjOutputDtype,
979
1167
  executionPolicies: state.executionPolicies ?? null,
1168
+ residualTensor: canUseWideTileResidual ? residualTensor : null,
980
1169
  });
1170
+ residualFused = canUseWideTileResidual;
981
1171
  }
982
1172
  // Release temporary buffer if we created it (original was not already on GPU)
983
1173
  if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
@@ -1111,6 +1301,18 @@ export async function runLayerAttentionGPU(
1111
1301
  if (attentionInputTemp && attentionInput?.buffer) {
1112
1302
  releaseOnce(attentionInput.buffer);
1113
1303
  }
1304
+ if (decoderKTemp?.buffer) {
1305
+ releaseOnce(decoderKTemp.buffer);
1306
+ }
1307
+ if (decoderVTemp?.buffer) {
1308
+ releaseOnce(decoderVTemp.buffer);
1309
+ }
1310
+ if (decoderKVState?.ownedBuffers) {
1311
+ for (const buffer of decoderKVState.ownedBuffers) {
1312
+ releaseOnce(buffer);
1313
+ }
1314
+ decoderKVState.ownedBuffers = null;
1315
+ }
1114
1316
  throw error;
1115
1317
  }
1116
1318
  }
@@ -39,8 +39,12 @@ export interface AttentionConfig {
39
39
  attnSoftcap?: number;
40
40
  /** Gemma 2 attention scaling: uses head_dim (256) instead of sqrt(head_dim) (16). */
41
41
  queryPreAttnScalar?: number;
42
- /** Apply query/key RMSNorm even when per-head weights are absent. */
42
+ /** Apply query/key RMSNorm where per-head weights are declared. */
43
43
  queryKeyNorm?: boolean;
44
+ /** Layers that apply query/key RMSNorm; null means all layers when queryKeyNorm=true. */
45
+ queryKeyNormLayers?: number[] | null;
46
+ /** Layers that carry explicit Q/K RMSNorm scale tensors; null means missing weights are invalid. */
47
+ queryKeyNormWeightLayers?: number[] | null;
44
48
  /** Apply unit-scale value RMSNorm before attention. */
45
49
  valueNorm?: boolean;
46
50
  /** Apply sigmoid gate from q_proj split to attention output before o_proj. */
@@ -109,9 +109,23 @@ export interface TensorInfo {
109
109
  dtype?: string;
110
110
  }
111
111
 
112
+ export interface ParsedSessionSettings {
113
+ prefillChunkSubmitMode: 'sync' | 'async' | null;
114
+ prefillTokenChunkSize: number | null;
115
+ useFlashPrefillAttention: boolean | null;
116
+ useWideTileQ4KPrefill: boolean | null;
117
+ useWideTileQ4KDecode: boolean | null;
118
+ useSandwichRMSNormPairFusion: boolean | null;
119
+ usePostFfnNextInputRMSNormPairFusion: boolean | null;
120
+ useFusedQKVSplitQKNorm: boolean | null;
121
+ useFusedQKVSplitQKNormRoPE: boolean | null;
122
+ retainQ4KMaterialization: boolean | null;
123
+ }
124
+
112
125
  export type Manifest = RuntimeModelContract;
113
126
 
114
127
  export interface ParsedModelConfig {
128
+ modelType: string;
115
129
  numLayers: number;
116
130
  hiddenSize: number;
117
131
  intermediateSize: number;
@@ -130,7 +144,8 @@ export interface ParsedModelConfig {
130
144
  useMoE: boolean;
131
145
  numExperts: number;
132
146
  moeTopK: number;
133
- expertFormat: 'mixtral' | 'gpt-oss' | null;
147
+ expertFormat: 'mixtral' | 'gpt-oss' | 'gemma4' | null;
148
+ moeExpertIntermediateSize: number;
134
149
  slidingWindow: number | null;
135
150
  ropeTheta: number;
136
151
  ropeLocalTheta: number | null;
@@ -162,6 +177,7 @@ export interface ParsedModelConfig {
162
177
  embeddingVocabSize: number | null;
163
178
  embeddingPostprocessor: ManifestEmbeddingPostprocessorSchema | null;
164
179
  hiddenActivation: ActivationType;
180
+ ffnBranchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
165
181
  useDoubleWideMlp: boolean;
166
182
  swigluLimit: number | null;
167
183
  stopTokenIds: number[];
@@ -178,6 +194,8 @@ export interface ParsedModelConfig {
178
194
  finalLogitSoftcapping: number | null;
179
195
  attnLogitSoftcapping: number | null;
180
196
  queryKeyNorm: boolean;
197
+ queryKeyNormLayers: number[] | null;
198
+ queryKeyNormWeightLayers: number[] | null;
181
199
  valueNorm: boolean;
182
200
  attentionOutputGate: boolean;
183
201
  queryPreAttnScalar: number;
@@ -186,7 +204,9 @@ export interface ParsedModelConfig {
186
204
  chatTemplateEnabled: boolean;
187
205
  chatTemplateThinking: boolean | null;
188
206
  decodeStrategy: 'incremental' | 'replay_prefill';
207
+ diffusionGemma: ManifestInferenceSchema['diffusionGemma'];
189
208
  perLayerInputsSession: ExecutionV1PerLayerInputsSessionSchema;
209
+ sessionSettings: ParsedSessionSettings;
190
210
  kernelPath?: KernelPathRef;
191
211
  visionConfig?: VisionConfig | null;
192
212
  audioConfig?: AudioEncoderConfig | null;
@@ -246,6 +266,7 @@ export interface AudioEncoderConfig {
246
266
 
247
267
  export function getStopTokenIds(manifest: Manifest): number[];
248
268
  export function resolveLayerIntermediateSize(config: ParsedModelConfig, layerIdx: number): number;
269
+ export function assertSupportedManifestInference(manifest: Manifest): void;
249
270
 
250
271
  /**
251
272
  * Extended manifest with inference config for manifest-first parsing.