doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -0,0 +1,8 @@
1
+ import type { LayerWeights } from '../types.js';
2
+
3
+ export declare function hasQ4KMaterialization(weight: unknown): boolean;
4
+
5
+ export declare function canUseRmsNormWideTileProjectionFusion(
6
+ layerWeights: LayerWeights | null | undefined,
7
+ reusesSharedKV: boolean
8
+ ): boolean;
@@ -0,0 +1,19 @@
1
+ import { isWeightBuffer } from '../../../../gpu/weight-buffer.js';
2
+
3
+ export function hasQ4KMaterialization(weight) {
4
+ return isWeightBuffer(weight) && Boolean(weight.materializations?.q4k?.buffer);
5
+ }
6
+
7
+ export function canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV) {
8
+ if (hasQ4KMaterialization(layerWeights?.qkvProj)) {
9
+ return true;
10
+ }
11
+ if (!hasQ4KMaterialization(layerWeights?.qProj)) {
12
+ return false;
13
+ }
14
+ if (reusesSharedKV) {
15
+ return true;
16
+ }
17
+ return hasQ4KMaterialization(layerWeights?.kProj)
18
+ && (layerWeights?.vProj == null || hasQ4KMaterialization(layerWeights.vProj));
19
+ }
@@ -1,12 +1,15 @@
1
1
 
2
2
 
3
3
  import { isGpuBufferInstance, isWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
4
- import { getDevice } from '../../../../gpu/device.js';
4
+ import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
5
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
5
6
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
6
7
  import {
7
8
  runMatmul,
8
9
  runRMSNorm,
9
10
  runRoPE,
11
+ canUseRoPEQK,
12
+ runRoPEQK,
10
13
  runAttention,
11
14
  runAttentionBDPA,
12
15
  runAttentionTieredQuant,
@@ -33,6 +36,9 @@ import {
33
36
  projectAttentionQKV,
34
37
  applyAttentionQKNorm,
35
38
  applyAttentionValueNorm,
39
+ hasAttentionProjectionDiagnostics,
40
+ hasAttentionStageDiagnostics,
41
+ resolveAttentionQKNormState,
36
42
  } from './projections.js';
37
43
  import { prepareAttentionProjectionInput } from './output-projection.js';
38
44
 
@@ -46,6 +52,7 @@ import {
46
52
  } from '../../../../config/kernel-path-loader.js';
47
53
  import {
48
54
  resolveKVCacheState,
55
+ createDiffusionGemmaDecoderKVState,
49
56
  buildAttentionDispatchParams,
50
57
  buildAttentionInputsData,
51
58
  } from './dispatch-params.js';
@@ -58,9 +65,30 @@ import {
58
65
  resolveAttentionPrecisionContract,
59
66
  isAttentionKvDtypeExplicit,
60
67
  } from './precision-contract.js';
68
+ import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
61
69
 
62
70
  const ATTENTION_DTYPE_LOGGED = new Set();
63
71
 
72
+ function isWideTileQ4KPhaseEnabled(session, phase) {
73
+ return phase === 'decode'
74
+ ? session?.useWideTileQ4KDecode === true
75
+ : session?.useWideTileQ4KPrefill === true;
76
+ }
77
+
78
+ function canUseQ4KWideTileResidualFusion(options = {}) {
79
+ return options.hasResidual === true
80
+ && options.oProjDtype === 'q4k'
81
+ && options.inputDtype === 'f32'
82
+ && options.outputDtype === 'f32'
83
+ && options.residualMatches === true
84
+ && options.hasLoRA !== true
85
+ && getKernelCapabilities().hasF16 === true
86
+ && getKernelCapabilities().hasSubgroups === true
87
+ && options.session?.useWideTileResidualFusion === true
88
+ && options.session?.retainQ4KMaterialization === true
89
+ && isWideTileQ4KPhaseEnabled(options.session, options.phase);
90
+ }
91
+
64
92
  function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
65
93
  assertImplicitDtypeTransitionAllowed({
66
94
  executionPolicies: state?.executionPolicies ?? null,
@@ -106,6 +134,8 @@ export async function runLayerAttentionGPU(
106
134
  multimodalBidirectionalSpan = null,
107
135
  sharedKVSourceLayerIdx = null,
108
136
  storeSharedKV = false,
137
+ diffusionGemmaDecoder = false,
138
+ precomputedInputNorm = null,
109
139
  } = config;
110
140
 
111
141
  const device = getDevice();
@@ -128,6 +158,9 @@ export async function runLayerAttentionGPU(
128
158
  const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
129
159
  const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
130
160
  const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
161
+ const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
162
+ ? 'step_precision'
163
+ : null;
131
164
  let attentionInput = input;
132
165
  let attentionInputTemp = false;
133
166
  let normed = attentionInput;
@@ -140,12 +173,17 @@ export async function runLayerAttentionGPU(
140
173
  let output = null;
141
174
  let finalOutput = null;
142
175
  let oProjInputTemp = null;
176
+ let valueAliasesKey = false;
177
+ let decoderKVState = null;
178
+ let decoderKTemp = null;
179
+ let decoderVTemp = null;
143
180
  if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
144
181
  assertAttentionDtypeTransitionAllowed(
145
182
  state,
146
183
  input.dtype,
147
184
  attentionActivationDtype,
148
- 'The attention kernel selection would widen the input implicitly.'
185
+ 'The attention kernel selection would widen the input implicitly.',
186
+ attentionInputTransitionDeclaredBy
149
187
  );
150
188
  attentionInput = attentionActivationDtype === 'f16'
151
189
  ? await castF32ToF16(input)
@@ -192,9 +230,57 @@ export async function runLayerAttentionGPU(
192
230
  const reusesSharedKV = sharedKVEntry != null;
193
231
 
194
232
  // 1. Input norm
195
-
233
+ // Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
234
+ // defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
235
+ // variant. Saves one standalone rmsnorm dispatch per layer.
236
+ let fusedNormWeight = null;
237
+ let fusedNormEps = null;
238
+ let fusedNormOffset = false;
239
+ let fusedNormWeightOwned = false;
240
+ // state.runtimeConfig.inference.session is a stale snapshot ({}); the live
241
+ // module-level runtime config carries the merged profile/override session.
242
+ const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
243
+ const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
244
+ const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
245
+ const canFuseInputNormIntoProjections = rmsNormFusionFlag
246
+ && hasInputNormWeight
247
+ && canSelectFusedRmsNormProjection
248
+ && !skipInputNorm
249
+ && isPrefill
250
+ && numTokens > 1
251
+ && attentionInput.dtype === 'f32';
252
+
196
253
  try {
197
- if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
254
+ if (precomputedInputNorm) {
255
+ if (
256
+ isPrefill
257
+ || numTokens !== 1
258
+ || skipInputNorm
259
+ || !layerWeights.inputNorm
260
+ || !getNormWeightBuffer
261
+ || precomputedInputNorm.dtype !== attentionInput.dtype
262
+ || precomputedInputNorm.dtype !== 'f32'
263
+ ) {
264
+ throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
265
+ }
266
+ normed = precomputedInputNorm;
267
+ await runProbes('post_input_norm', normed.buffer, {
268
+ layerIdx,
269
+ numTokens,
270
+ hiddenSize,
271
+ probes: state.debugProbes,
272
+ operatorDiagnostics: state.operatorDiagnostics,
273
+ dtype: normed.dtype,
274
+ });
275
+ } else if (canFuseInputNormIntoProjections) {
276
+ const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
277
+ fusedNormWeight = normWeightBuf;
278
+ fusedNormEps = rmsNormEps;
279
+ fusedNormOffset = config.rmsNormWeightOffset === true;
280
+ fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
281
+ // Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
282
+ // norm internally via the fused kernel variant.
283
+ } else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
198
284
  const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
199
285
  try {
200
286
  // Debug: norm weights for configured layers
@@ -275,7 +361,24 @@ export async function runLayerAttentionGPU(
275
361
  }),
276
362
  });
277
363
  let usedFusedQKV = false;
278
- ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
364
+ let qkNormApplied = false;
365
+ let ropeApplied = false;
366
+ const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
367
+ const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
368
+ || kernelTrace.enabled
369
+ || isKernelDebugEnabled(layerIdx)
370
+ || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
371
+ const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
372
+ state,
373
+ ['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
374
+ )
375
+ || kernelTrace.enabled
376
+ || isKernelDebugEnabled(layerIdx)
377
+ || (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
378
+ const runtimeSession = getRuntimeConfig()?.inference?.session;
379
+ const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
380
+ const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
381
+ ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
279
382
  recorder: null,
280
383
  normed,
281
384
  layerWeights,
@@ -300,7 +403,43 @@ export async function runLayerAttentionGPU(
300
403
  trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
301
404
  }
302
405
  : null,
406
+ fusedNormWeight,
407
+ fusedNormEps,
408
+ fusedNormOffset,
409
+ qkNormFusion: {
410
+ enabled: qkNormFusionFlag && qkNormState.wantsQKNorm,
411
+ getNormWeightBuffer,
412
+ rmsNormEps,
413
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
414
+ skipKNorm: qkNormState.skipKNorm,
415
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
416
+ projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
417
+ },
418
+ qkNormRoPEFusion: {
419
+ enabled: qkNormRoPEFusionFlag
420
+ && qkNormState.wantsQKNorm
421
+ && !disableRoPE
422
+ && !!state.ropeFreqsCos
423
+ && !!state.ropeFreqsSin,
424
+ getNormWeightBuffer,
425
+ rmsNormEps,
426
+ rmsNormWeightOffset: config.rmsNormWeightOffset,
427
+ skipKNorm: qkNormState.skipKNorm,
428
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
429
+ projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
430
+ freqsCos: state.ropeFreqsCos,
431
+ freqsSin: state.ropeFreqsSin,
432
+ startPos: currentSeqLen,
433
+ rotaryDim: config.ropeRotaryDim,
434
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
435
+ interleaved: config.ropeInterleaved,
436
+ reusesSharedKV,
437
+ },
303
438
  }));
439
+ // Release the in-flight norm weight buffer used by the fused projections.
440
+ if (fusedNormWeight && fusedNormWeightOwned) {
441
+ releaseBuffer(fusedNormWeight);
442
+ }
304
443
 
305
444
  // Trace Q/K/V projections
306
445
  if (kernelTrace.enabled) {
@@ -369,17 +508,19 @@ export async function runLayerAttentionGPU(
369
508
  }
370
509
 
371
510
  // Optional per-head Q/K normalization
372
- const wantsQKNorm = config.queryKeyNorm === true;
373
- const hasQNorm = !!layerWeights.qNorm;
374
- const hasKNorm = !!layerWeights.kNorm;
375
511
  if (isKernelDebugEnabled(layerIdx)) {
376
- logKernelStep('qk_norm', { layerIdx, label: `hasQ=${hasQNorm} hasK=${hasKNorm} wants=${wantsQKNorm}` });
377
- }
378
- if (wantsQKNorm && layerIdx === 0 && (!hasQNorm || !hasKNorm)) {
379
- log.warn('Attention', `Q/K norm requested but weights missing (hasQ=${hasQNorm}, hasK=${hasKNorm}); skipping QK norm.`);
512
+ logKernelStep(
513
+ 'qk_norm',
514
+ {
515
+ layerIdx,
516
+ label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
517
+ `wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
518
+ }
519
+ );
380
520
  }
381
521
 
382
- ({ qTensor, kTensor } = await applyAttentionQKNorm({
522
+ if (qkNormState.wantsQKNorm && !qkNormApplied) {
523
+ ({ qTensor, kTensor } = await applyAttentionQKNorm({
383
524
  recorder: null,
384
525
  qTensor,
385
526
  kTensor,
@@ -392,7 +533,7 @@ export async function runLayerAttentionGPU(
392
533
  headDim,
393
534
  rmsNormWeightOffset: config.rmsNormWeightOffset,
394
535
  releaseTemporary: (buffer) => releaseBuffer(buffer),
395
- skipKNorm: reusesSharedKV,
536
+ skipKNorm: qkNormState.skipKNorm,
396
537
  onQNormApplied: isKernelDebugEnabled(layerIdx)
397
538
  ? async (tensor) => {
398
539
  await dumpTokenVector(tensor.buffer, 'Q_norm', {
@@ -412,10 +553,14 @@ export async function runLayerAttentionGPU(
412
553
  dtype: tensor.dtype,
413
554
  });
414
555
  }
415
- : null,
416
- }));
556
+ : null,
557
+ retainKInput: valueAliasesKey,
558
+ allowUnitQKNorm: qkNormState.allowUnitQKNorm,
559
+ }));
560
+ }
417
561
 
418
562
  if (config.valueNorm === true && !reusesSharedKV) {
563
+ const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
419
564
  vTensor = await applyAttentionValueNorm({
420
565
  recorder: null,
421
566
  vTensor,
@@ -423,27 +568,41 @@ export async function runLayerAttentionGPU(
423
568
  numTokens,
424
569
  numKVHeads,
425
570
  headDim,
426
- releaseTemporary: (buffer) => releaseBuffer(buffer),
571
+ releaseTemporary: (buffer) => {
572
+ if (!valueNormInputAliasesKey) {
573
+ releaseBuffer(buffer);
574
+ }
575
+ },
576
+ });
577
+ await runProbes('v_norm', vTensor.buffer, {
578
+ layerIdx,
579
+ numTokens,
580
+ hiddenSize: numKVHeads * headDim,
581
+ probes: state.debugProbes,
582
+ operatorDiagnostics: state.operatorDiagnostics,
583
+ dtype: vTensor.dtype,
427
584
  });
428
585
  }
429
586
 
430
- await runProbes('q_norm', qTensor.buffer, {
431
- layerIdx,
432
- numTokens,
433
- hiddenSize: numHeads * headDim,
434
- probes: state.debugProbes,
435
- operatorDiagnostics: state.operatorDiagnostics,
436
- dtype: qTensor.dtype,
437
- });
438
- if (!reusesSharedKV) {
439
- await runProbes('k_norm', kTensor.buffer, {
587
+ if (qkNormState.wantsQKNorm) {
588
+ await runProbes('q_norm', qTensor.buffer, {
440
589
  layerIdx,
441
590
  numTokens,
442
- hiddenSize: numKVHeads * headDim,
591
+ hiddenSize: numHeads * headDim,
443
592
  probes: state.debugProbes,
444
593
  operatorDiagnostics: state.operatorDiagnostics,
445
- dtype: kTensor.dtype,
594
+ dtype: qTensor.dtype,
446
595
  });
596
+ if (!reusesSharedKV) {
597
+ await runProbes('k_norm', kTensor.buffer, {
598
+ layerIdx,
599
+ numTokens,
600
+ hiddenSize: numKVHeads * headDim,
601
+ probes: state.debugProbes,
602
+ operatorDiagnostics: state.operatorDiagnostics,
603
+ dtype: kTensor.dtype,
604
+ });
605
+ }
447
606
  }
448
607
 
449
608
  if (normed !== attentionInput) releaseBuffer(normed.buffer);
@@ -451,24 +610,32 @@ export async function runLayerAttentionGPU(
451
610
 
452
611
  // 3. RoPE (modifies tensor in-place)
453
612
 
454
- if (!disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
455
- await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
456
- numHeads,
613
+ if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
614
+ const ropeOptions = {
457
615
  headDim,
458
616
  rotaryDim: config.ropeRotaryDim,
617
+ pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
459
618
  interleaved: config.ropeInterleaved,
460
619
  startPos: currentSeqLen,
461
620
  executionPolicies: state.executionPolicies ?? null,
462
- });
463
- if (!reusesSharedKV) {
464
- await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
465
- numHeads: numKVHeads,
466
- headDim,
467
- rotaryDim: config.ropeRotaryDim,
468
- interleaved: config.ropeInterleaved,
469
- startPos: currentSeqLen,
470
- executionPolicies: state.executionPolicies ?? null,
621
+ };
622
+ if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
623
+ await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
624
+ ...ropeOptions,
625
+ numQHeads: numHeads,
626
+ numKVHeads,
627
+ });
628
+ } else {
629
+ await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
630
+ ...ropeOptions,
631
+ numHeads,
471
632
  });
633
+ if (!reusesSharedKV) {
634
+ await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
635
+ ...ropeOptions,
636
+ numHeads: numKVHeads,
637
+ });
638
+ }
472
639
  }
473
640
 
474
641
  // Trace RoPE outputs
@@ -529,7 +696,7 @@ export async function runLayerAttentionGPU(
529
696
 
530
697
  // 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
531
698
 
532
- if (state.kvCache?.hasGPUCache?.()) {
699
+ if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
533
700
  if (state.kvCache.kvDtype === 'f16') {
534
701
  const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
535
702
  if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
@@ -557,11 +724,66 @@ export async function runLayerAttentionGPU(
557
724
  }
558
725
 
559
726
  // Resolve KV cache state and build dispatch parameters (shared with record.js)
560
- const kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
727
+ let kvState;
728
+ if (diffusionGemmaDecoder) {
729
+ const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
730
+ let decoderKTensor = kTensor;
731
+ let decoderVTensor = vTensor;
732
+ if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
733
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
734
+ if (!hasExplicitF16KvContract) {
735
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
736
+ }
737
+ decoderKTensor = await castF32ToF16(decoderKTensor);
738
+ decoderKTemp = decoderKTensor;
739
+ } else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
740
+ assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
741
+ decoderKTensor = await castF16ToF32(decoderKTensor);
742
+ decoderKTemp = decoderKTensor;
743
+ }
744
+ if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
745
+ const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
746
+ if (!hasExplicitF16KvContract) {
747
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
748
+ }
749
+ decoderVTensor = await castF32ToF16(decoderVTensor);
750
+ decoderVTemp = decoderVTensor;
751
+ } else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
752
+ assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
753
+ decoderVTensor = await castF16ToF32(decoderVTensor);
754
+ decoderVTemp = decoderVTensor;
755
+ }
756
+ kvState = await createDiffusionGemmaDecoderKVState({
757
+ state,
758
+ layerIdx,
759
+ kTensor: decoderKTensor,
760
+ vTensor: decoderVTensor,
761
+ currentSeqLen,
762
+ numTokens,
763
+ numKVHeads,
764
+ headDim,
765
+ layerType,
766
+ slidingWindow,
767
+ kvDtype: decoderKVDtype,
768
+ });
769
+ if (decoderKTemp) {
770
+ releaseBuffer(decoderKTemp.buffer);
771
+ decoderKTemp = null;
772
+ }
773
+ if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
774
+ releaseBuffer(decoderVTemp.buffer);
775
+ }
776
+ decoderVTemp = null;
777
+ } else {
778
+ kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
779
+ }
561
780
  const dispatchConfig = {
562
781
  layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
563
- slidingWindow, layerType, layerTypes: config.layerTypes,
564
- queryPreAttnScalar, causalAttention: config.causalAttention,
782
+ slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
783
+ layerType,
784
+ layerTypes: config.layerTypes,
785
+ queryPreAttnScalar,
786
+ causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
565
787
  activationDtype: attentionActivationDtype,
566
788
  kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
567
789
  };
@@ -763,6 +985,15 @@ export async function runLayerAttentionGPU(
763
985
  kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
764
986
  vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
765
987
  }
988
+ // Session precedence is runtime-over-manifest per config-style-guide
989
+ // §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
990
+ // with runtime fields winning. getRuntimeConfig() returns that merged
991
+ // session (state.runtimeConfig.inference.session is a stale snapshot —
992
+ // see comment above at the rmsNorm fusion check). The kernel itself
993
+ // enforces remaining preconditions (head_dim=256, etc.).
994
+ const mergedSession = getRuntimeConfig()?.inference?.session;
995
+ const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
996
+ const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
766
997
  const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
767
998
  seqLen: numTokens,
768
999
  kvLen: kvState.kvLenForAttention,
@@ -780,6 +1011,8 @@ export async function runLayerAttentionGPU(
780
1011
  kvPageTable: kvState.kvPageTable,
781
1012
  kvPageSize: kvState.kvPageSize,
782
1013
  kernelPath,
1014
+ useFlashPrefill,
1015
+ useOrtFlashPrefill,
783
1016
  });
784
1017
  if (prefillFallbackNeedsCast) {
785
1018
  if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
@@ -794,7 +1027,24 @@ export async function runLayerAttentionGPU(
794
1027
  throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
795
1028
  }
796
1029
 
797
- attnOutput = await runAttentionKernel();
1030
+ try {
1031
+ attnOutput = await runAttentionKernel();
1032
+ } finally {
1033
+ if (decoderKVState?.ownedBuffers) {
1034
+ for (const buffer of decoderKVState.ownedBuffers) {
1035
+ releaseBuffer(buffer);
1036
+ }
1037
+ decoderKVState.ownedBuffers = null;
1038
+ }
1039
+ }
1040
+ await runProbes('attn_core_out', attnOutput.buffer, {
1041
+ layerIdx,
1042
+ numTokens,
1043
+ hiddenSize: numHeads * headDim,
1044
+ probes: state.debugProbes,
1045
+ operatorDiagnostics: state.operatorDiagnostics,
1046
+ dtype: attnOutput.dtype,
1047
+ });
798
1048
 
799
1049
  // Trace attention output
800
1050
  if (kernelTrace.enabled) {
@@ -819,10 +1069,27 @@ export async function runLayerAttentionGPU(
819
1069
 
820
1070
  attnForProjection = attnOutput;
821
1071
  if (qGateTensor) {
1072
+ // The Qwen3_5 reference (HF transformers) full-attention path
1073
+ // applies sigmoid(gate) — NOT silu(gate) — to the attention
1074
+ // output regardless of how the architecture's gate label is
1075
+ // surfaced in the HF config. See Qwen3_5Attention.forward in
1076
+ // transformers/models/qwen3_5/modeling_qwen3_5.py:
1077
+ // attn_output = attn_output * torch.sigmoid(gate)
1078
+ // The earlier `swish` -> `silu` mapping here was incorrect and
1079
+ // produced sign-flipped output for any token whose gate value
1080
+ // was negative (silu is sign-preserving via x * sigmoid(x);
1081
+ // sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
1082
+ // this drove residual-stream divergence by L=3 (HF reference vs
1083
+ // upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
1084
+ // sign-flip pattern). config.outputGateType is retained for
1085
+ // future architectures whose gate truly is silu, but Qwen 3.6's
1086
+ // `output_gate_type=swish` still routes here as sigmoid because
1087
+ // that is what the reference forward computes.
1088
+ const gateActivation = 'sigmoid';
822
1089
  attnForProjection = await runSiLU(attnOutput, {
823
1090
  size: numTokens * numHeads * headDim,
824
1091
  gate: qGateTensor,
825
- gateActivation: 'sigmoid',
1092
+ gateActivation,
826
1093
  inputActivation: 'identity',
827
1094
  swigluLimit: null,
828
1095
  });
@@ -858,6 +1125,7 @@ export async function runLayerAttentionGPU(
858
1125
  // Use fused o_proj + residual for decode when possible
859
1126
  // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
860
1127
  const oProjDtype = getWeightDtype(oProjBuf);
1128
+ const mergedSession = getRuntimeConfig()?.inference?.session;
861
1129
  const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
862
1130
  allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
863
1131
  hasResidual: Boolean(residualTensor),
@@ -867,7 +1135,16 @@ export async function runLayerAttentionGPU(
867
1135
  hasLoRA: Boolean(loraO),
868
1136
  oProjIsF16: oProjDtype === 'f16',
869
1137
  }); // GEMV kernel expects f16 weights
870
-
1138
+ const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
1139
+ phase,
1140
+ session: mergedSession,
1141
+ hasResidual: Boolean(residualTensor),
1142
+ residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
1143
+ inputDtype: oProjInput.dtype,
1144
+ outputDtype: oProjOutputDtype,
1145
+ oProjDtype,
1146
+ hasLoRA: Boolean(loraO),
1147
+ });
871
1148
  if (canUseFused && residualTensor) {
872
1149
  // FUSED PATH: o_proj matmul + residual add in one dispatch
873
1150
  output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
@@ -880,7 +1157,7 @@ export async function runLayerAttentionGPU(
880
1157
  trace.attn(layerIdx, `Using fused o_proj+residual path`);
881
1158
  }
882
1159
  } else {
883
- // STANDARD PATH: o_proj matmul only (residual will be added by layer.ts)
1160
+ // STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
884
1161
  output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
885
1162
  transposeB: 'auto',
886
1163
  role: 'o_proj',
@@ -888,7 +1165,9 @@ export async function runLayerAttentionGPU(
888
1165
  kernelPath,
889
1166
  outputDtype: oProjOutputDtype,
890
1167
  executionPolicies: state.executionPolicies ?? null,
1168
+ residualTensor: canUseWideTileResidual ? residualTensor : null,
891
1169
  });
1170
+ residualFused = canUseWideTileResidual;
892
1171
  }
893
1172
  // Release temporary buffer if we created it (original was not already on GPU)
894
1173
  if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
@@ -973,7 +1252,9 @@ export async function runLayerAttentionGPU(
973
1252
  }
974
1253
  if (!reusesSharedKV && !storeSharedKV) {
975
1254
  releaseBuffer(kTensor.buffer);
976
- releaseBuffer(vTensor.buffer);
1255
+ if (vTensor.buffer !== kTensor.buffer) {
1256
+ releaseBuffer(vTensor.buffer);
1257
+ }
977
1258
  }
978
1259
  for (const buffer of buffersToRelease) {
979
1260
  releaseBuffer(buffer);
@@ -1020,6 +1301,18 @@ export async function runLayerAttentionGPU(
1020
1301
  if (attentionInputTemp && attentionInput?.buffer) {
1021
1302
  releaseOnce(attentionInput.buffer);
1022
1303
  }
1304
+ if (decoderKTemp?.buffer) {
1305
+ releaseOnce(decoderKTemp.buffer);
1306
+ }
1307
+ if (decoderVTemp?.buffer) {
1308
+ releaseOnce(decoderVTemp.buffer);
1309
+ }
1310
+ if (decoderKVState?.ownedBuffers) {
1311
+ for (const buffer of decoderKVState.ownedBuffers) {
1312
+ releaseOnce(buffer);
1313
+ }
1314
+ decoderKVState.ownedBuffers = null;
1315
+ }
1023
1316
  throw error;
1024
1317
  }
1025
1318
  }
@@ -39,8 +39,12 @@ export interface AttentionConfig {
39
39
  attnSoftcap?: number;
40
40
  /** Gemma 2 attention scaling: uses head_dim (256) instead of sqrt(head_dim) (16). */
41
41
  queryPreAttnScalar?: number;
42
- /** Apply query/key RMSNorm even when per-head weights are absent. */
42
+ /** Apply query/key RMSNorm where per-head weights are declared. */
43
43
  queryKeyNorm?: boolean;
44
+ /** Layers that apply query/key RMSNorm; null means all layers when queryKeyNorm=true. */
45
+ queryKeyNormLayers?: number[] | null;
46
+ /** Layers that carry explicit Q/K RMSNorm scale tensors; null means missing weights are invalid. */
47
+ queryKeyNormWeightLayers?: number[] | null;
44
48
  /** Apply unit-scale value RMSNorm before attention. */
45
49
  valueNorm?: boolean;
46
50
  /** Apply sigmoid gate from q_proj split to attention output before o_proj. */
@@ -56,6 +60,8 @@ export interface AttentionConfig {
56
60
  } | null;
57
61
  /** RoPE rotary dimension (may differ from headDim with partial rotary). */
58
62
  ropeRotaryDim?: number;
63
+ /** RoPE frequency base dimension; non-interleaved kernels use this as the rotate-half pair span. */
64
+ ropeFrequencyBaseDim?: number;
59
65
  /** Whether RoPE uses interleaved layout. */
60
66
  ropeInterleaved?: boolean;
61
67
  /** Token IDs for the current micro-batch (required by BDPA KV ingestion). */
@@ -36,18 +36,8 @@ export type ChatTemplateType =
36
36
  | 'translategemma'
37
37
  | null;
38
38
 
39
- export declare function formatGemmaChat(messages: ChatMessage[]): string;
40
-
41
39
  export interface ChatFormatOptions {
42
40
  thinking?: boolean;
43
41
  }
44
42
 
45
- export declare function formatGemma4Chat(messages: ChatMessage[], options?: ChatFormatOptions): string;
46
-
47
- export declare function formatLlama3Chat(messages: ChatMessage[]): string;
48
-
49
- export declare function formatGptOssChat(messages: ChatMessage[]): string;
50
-
51
- export declare function formatTranslateGemmaChat(messages: ChatMessage[]): string;
52
-
53
43
  export declare function formatChatMessages(messages: ChatMessage[], templateType?: ChatTemplateType, options?: ChatFormatOptions): string;