doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -72,6 +72,69 @@ export function resolveProjectionSliceOffsetBytes(
72
72
  inputCols: number
73
73
  ): number;
74
74
 
75
+ export interface AttentionQKNormState {
76
+ wantsQKNorm: boolean;
77
+ hasQNorm: boolean;
78
+ hasKNorm: boolean;
79
+ allowUnitQKNorm: boolean;
80
+ skipKNorm: boolean;
81
+ }
82
+
83
+ export function hasAttentionProjectionDiagnostics(state: {
84
+ operatorDiagnostics?: {
85
+ enabled?: boolean;
86
+ tsirFixture?: { dir?: string | null } | null;
87
+ } | null;
88
+ debugProbes?: Array<{ stage?: string | null }> | null;
89
+ } | null | undefined): boolean;
90
+
91
+ export function hasAttentionStageDiagnostics(
92
+ state: {
93
+ operatorDiagnostics?: {
94
+ enabled?: boolean;
95
+ tsirFixture?: { dir?: string | null } | null;
96
+ } | null;
97
+ debugProbes?: Array<{ stage?: string | null }> | null;
98
+ } | null | undefined,
99
+ stages: string[]
100
+ ): boolean;
101
+
102
+ export function resolveAttentionQKNormState(options: {
103
+ config: {
104
+ queryKeyNorm?: boolean;
105
+ queryKeyNormWeightLayers?: number[] | null;
106
+ };
107
+ layerWeights: LayerWeights;
108
+ layerIdx: number;
109
+ reusesSharedKV: boolean;
110
+ }): AttentionQKNormState;
111
+
112
+ export interface ProjectAttentionQKNormFusionOptions {
113
+ enabled: boolean;
114
+ getNormWeightBuffer?: (weight: GPUBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer, label: string) => GPUBuffer;
115
+ rmsNormEps: number;
116
+ rmsNormWeightOffset?: boolean;
117
+ skipKNorm?: boolean;
118
+ allowUnitQKNorm?: boolean;
119
+ projectionDiagnosticsEnabled?: boolean;
120
+ }
121
+
122
+ export interface ProjectAttentionQKNormRoPEFusionOptions extends ProjectAttentionQKNormFusionOptions {
123
+ freqsCos?: GPUBuffer | Tensor | null;
124
+ freqsSin?: GPUBuffer | Tensor | null;
125
+ headDim?: number;
126
+ startPos?: number;
127
+ rotaryDim?: number;
128
+ pairSpanDim?: number;
129
+ interleaved?: boolean;
130
+ reusesSharedKV?: boolean;
131
+ f16KVCacheWrite?: {
132
+ keysBuffer: GPUBuffer;
133
+ valuesBuffer: GPUBuffer;
134
+ dstOffset: number;
135
+ } | null;
136
+ }
137
+
75
138
  export interface ProjectAttentionQKVOptions {
76
139
  recorder?: CommandRecorder | null;
77
140
  normed: Tensor;
@@ -88,15 +151,20 @@ export interface ProjectAttentionQKVOptions {
88
151
  releaseTemporary: (buffer: GPUBuffer) => void;
89
152
  matmulDebug?: MatmulDebugConfigSchema | null;
90
153
  onFusedQKV?: ((info: { qSize: number; kSize: number; vSize: number; totalSize: number }) => void) | null;
154
+ qkNormFusion?: ProjectAttentionQKNormFusionOptions | null;
155
+ qkNormRoPEFusion?: ProjectAttentionQKNormRoPEFusionOptions | null;
91
156
  }
92
157
 
93
158
  export interface ProjectAttentionQKVResult {
94
159
  qTensor: Tensor;
95
160
  qGateTensor: Tensor | null;
96
- kTensor: Tensor;
97
- vTensor: Tensor;
161
+ kTensor: Tensor | null;
162
+ vTensor: Tensor | null;
98
163
  usedFusedQKV: boolean;
99
164
  valueAliasesKey: boolean;
165
+ qkNormApplied: boolean;
166
+ ropeApplied: boolean;
167
+ kvCacheWriteFused: boolean;
100
168
  }
101
169
 
102
170
  export function projectAttentionQKV(options: ProjectAttentionQKVOptions): Promise<ProjectAttentionQKVResult>;
@@ -117,6 +185,7 @@ export interface ApplyAttentionQKNormOptions {
117
185
  onQNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
118
186
  onKNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
119
187
  retainKInput?: boolean;
188
+ allowUnitQKNorm?: boolean;
120
189
  }
121
190
 
122
191
  export function applyAttentionQKNorm(
@@ -9,6 +9,15 @@ import {
9
9
  recordSplitQG,
10
10
  runRMSNorm,
11
11
  recordRMSNorm,
12
+ canUseRMSNormQK,
13
+ runRMSNormQK,
14
+ recordRMSNormQK,
15
+ canUseSplitQKVRMSNormQK,
16
+ runSplitQKVRMSNormQK,
17
+ recordSplitQKVRMSNormQK,
18
+ canUseSplitQKVRMSNormRoPEQK,
19
+ runSplitQKVRMSNormRoPEQK,
20
+ recordSplitQKVRMSNormRoPEQK,
12
21
  castF16ToF32,
13
22
  castF32ToF16,
14
23
  recordCastF16ToF32,
@@ -50,6 +59,51 @@ function getRmsNormRunner(recorder) {
50
59
  return (input, weight, eps, options) => recordRMSNorm(recorder, input, weight, eps, options);
51
60
  }
52
61
 
62
+ function getRmsNormQKRunner(recorder) {
63
+ if (!recorder) {
64
+ return (q, k, qWeight, kWeight, eps, options) => runRMSNormQK(q, k, qWeight, kWeight, eps, options);
65
+ }
66
+ return (q, k, qWeight, kWeight, eps, options) => recordRMSNormQK(recorder, q, k, qWeight, kWeight, eps, options);
67
+ }
68
+
69
+ function getSplitQKVRMSNormQKRunner(recorder) {
70
+ if (!recorder) {
71
+ return (qkvTensor, qWeight, kWeight, eps, options) => runSplitQKVRMSNormQK(qkvTensor, qWeight, kWeight, eps, options);
72
+ }
73
+ return (qkvTensor, qWeight, kWeight, eps, options) => recordSplitQKVRMSNormQK(
74
+ recorder,
75
+ qkvTensor,
76
+ qWeight,
77
+ kWeight,
78
+ eps,
79
+ options
80
+ );
81
+ }
82
+
83
+ function getSplitQKVRMSNormRoPEQKRunner(recorder) {
84
+ if (!recorder) {
85
+ return (qkvTensor, qWeight, kWeight, freqsCos, freqsSin, eps, options) => runSplitQKVRMSNormRoPEQK(
86
+ qkvTensor,
87
+ qWeight,
88
+ kWeight,
89
+ freqsCos,
90
+ freqsSin,
91
+ eps,
92
+ options
93
+ );
94
+ }
95
+ return (qkvTensor, qWeight, kWeight, freqsCos, freqsSin, eps, options) => recordSplitQKVRMSNormRoPEQK(
96
+ recorder,
97
+ qkvTensor,
98
+ qWeight,
99
+ kWeight,
100
+ freqsCos,
101
+ freqsSin,
102
+ eps,
103
+ options
104
+ );
105
+ }
106
+
53
107
  function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemporary) {
54
108
  if (isGpuBufferInstance(layerWeight) || isWeightBuffer(layerWeight)) {
55
109
  return;
@@ -61,6 +115,73 @@ function releaseOwnedWeightBuffer(layerWeight, resolvedWeightBuffer, releaseTemp
61
115
  releaseTemporary(buffer);
62
116
  }
63
117
 
118
+ function normBufferMatchesHeadDim(buffer, headDim) {
119
+ if (!buffer || !Number.isFinite(buffer.size)) {
120
+ return false;
121
+ }
122
+ const elemsF32 = buffer.size / 4;
123
+ const elemsF16 = buffer.size / 2;
124
+ return elemsF32 === headDim || elemsF16 === headDim;
125
+ }
126
+
127
+ function ownsNormBuffer(layerWeight) {
128
+ return layerWeight && !isGpuBufferInstance(layerWeight) && !isWeightBuffer(layerWeight);
129
+ }
130
+
131
+ function releaseOwnedNormBuffer(buffer, owned, releaseTemporary, releasedBuffers) {
132
+ if (!owned || !buffer || releasedBuffers.has(buffer)) {
133
+ return;
134
+ }
135
+ releasedBuffers.add(buffer);
136
+ releaseTemporary(buffer);
137
+ }
138
+
139
+ export function hasAttentionProjectionDiagnostics(state) {
140
+ return hasAttentionStageDiagnostics(state, ['q_proj', 'k_proj', 'v_proj']);
141
+ }
142
+
143
+ export function hasAttentionStageDiagnostics(state, stages) {
144
+ const diagnostics = state?.operatorDiagnostics ?? null;
145
+ if (diagnostics?.enabled || diagnostics?.tsirFixture?.dir) {
146
+ return true;
147
+ }
148
+ const stageSet = new Set(stages);
149
+ const probes = state?.debugProbes;
150
+ return Array.isArray(probes) && probes.some((probe) => stageSet.has(probe?.stage));
151
+ }
152
+
153
+ export function resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV }) {
154
+ const wantsQKNorm = config.queryKeyNorm === true;
155
+ const hasQNorm = !!layerWeights.qNorm;
156
+ const hasKNorm = !!layerWeights.kNorm;
157
+ const qkNormWeightLayers = Array.isArray(config.queryKeyNormWeightLayers)
158
+ ? config.queryKeyNormWeightLayers
159
+ : null;
160
+ const expectsWeightedQKNorm = qkNormWeightLayers
161
+ ? qkNormWeightLayers.includes(layerIdx)
162
+ : true;
163
+ const allowUnitQKNorm = wantsQKNorm && qkNormWeightLayers !== null && !expectsWeightedQKNorm;
164
+ if (wantsQKNorm && allowUnitQKNorm && (hasQNorm || hasKNorm)) {
165
+ throw new Error(
166
+ `Layer ${layerIdx} declares unit-scale Q/K norm but companion weights are present ` +
167
+ `(hasQ=${hasQNorm}, hasK=${hasKNorm}). Check manifest.inference.attention.queryKeyNormWeightLayers.`
168
+ );
169
+ }
170
+ if (wantsQKNorm && expectsWeightedQKNorm && (!hasQNorm || (!hasKNorm && !reusesSharedKV))) {
171
+ throw new Error(
172
+ `Layer ${layerIdx} requested Q/K norm but companion weights are missing ` +
173
+ `(hasQ=${hasQNorm}, hasK=${hasKNorm}). Check manifest.inference.attention.queryKeyNormWeightLayers.`
174
+ );
175
+ }
176
+ return {
177
+ wantsQKNorm,
178
+ hasQNorm,
179
+ hasKNorm,
180
+ allowUnitQKNorm,
181
+ skipKNorm: reusesSharedKV,
182
+ };
183
+ }
184
+
64
185
  function normalizeProjectionMatmulDtype(value, precisionField = 'dtype') {
65
186
  if (value == null || value === '') {
66
187
  return null;
@@ -332,6 +453,23 @@ async function projectQueryWithOptionalGate({
332
453
  executionPolicies,
333
454
  });
334
455
 
456
+ const loraModule = getLoRAModule(lora, layerIdx, 'q_proj');
457
+ if (loraModule && getWeightBuffer) {
458
+ const combined = await applyLoRA(
459
+ normed,
460
+ fullQGTensor,
461
+ loraModule,
462
+ { M: numTokens, N: qSize * 2, K: hiddenSize },
463
+ getWeightBuffer,
464
+ recorder ?? undefined,
465
+ { kernelPath }
466
+ );
467
+ if (combined.buffer !== fullQGTensor.buffer) {
468
+ releaseTemporary(fullQGTensor.buffer);
469
+ fullQGTensor = combined;
470
+ }
471
+ }
472
+
335
473
  const split = await runSplitQGForMode(fullQGTensor, {
336
474
  numTokens,
337
475
  numHeads,
@@ -356,33 +494,6 @@ async function projectQueryWithOptionalGate({
356
494
  releaseOwnedWeightBuffer(qWeight, qWeightBuffer, releaseTemporary);
357
495
  }
358
496
 
359
- const loraModule = getLoRAModule(lora, layerIdx, 'q_proj');
360
- if (loraModule && getWeightBuffer) {
361
- try {
362
- const combined = await applyLoRA(
363
- normed,
364
- qTensor,
365
- loraModule,
366
- { M: numTokens, N: qSize, K: hiddenSize },
367
- getWeightBuffer,
368
- recorder ?? undefined,
369
- { kernelPath }
370
- );
371
- if (combined.buffer !== qTensor.buffer) {
372
- releaseTemporary(qTensor.buffer);
373
- qTensor = combined;
374
- }
375
- } catch (error) {
376
- if (qTensor?.buffer) {
377
- releaseTemporary(qTensor.buffer);
378
- }
379
- if (qGateTensor?.buffer) {
380
- releaseTemporary(qGateTensor.buffer);
381
- }
382
- throw error;
383
- }
384
- }
385
-
386
497
  return { qTensor, qGateTensor };
387
498
  }
388
499
 
@@ -448,9 +559,13 @@ export async function projectAttentionQKV({
448
559
  fusedNormWeight = null,
449
560
  fusedNormEps = null,
450
561
  fusedNormOffset = false,
562
+ qkNormFusion = null,
563
+ qkNormRoPEFusion = null,
451
564
  }) {
452
565
  const runMatmulForMode = getMatmulRunner(recorder);
453
566
  const runSplitForMode = getSplitRunner(recorder);
567
+ const runSplitQKVRMSNormQKForMode = getSplitQKVRMSNormQKRunner(recorder);
568
+ const runSplitQKVRMSNormRoPEQKForMode = getSplitQKVRMSNormRoPEQKRunner(recorder);
454
569
  const reuseSharedKV = sharedKTensor != null || sharedVTensor != null;
455
570
  if (reuseSharedKV && (!sharedKTensor || !sharedVTensor)) {
456
571
  throw new Error('projectAttentionQKV requires both sharedKTensor and sharedVTensor when reusing shared KV.');
@@ -491,6 +606,9 @@ export async function projectAttentionQKV({
491
606
  const [qSizeFused, kSizeFused, vSizeFused] = layerWeights.qkvSizes;
492
607
  const qkvSizeTotal = qSizeFused + kSizeFused + vSizeFused;
493
608
  let qkvTensor = null;
609
+ let qNormBuf = null;
610
+ let kNormBuf = null;
611
+ const releasedNormBuffers = new Set();
494
612
  try {
495
613
  qkvTensor = await runMatmulForMode(projectionInput, layerWeights.qkvProj, numTokens, qkvSizeTotal, hiddenSize, {
496
614
  transposeB: 'auto',
@@ -507,6 +625,114 @@ export async function projectAttentionQKV({
507
625
  rmsNormEps: fusedNormEps,
508
626
  rmsNormOffset: fusedNormOffset,
509
627
  });
628
+ const qkNormRoPEOptions = qkNormRoPEFusion
629
+ ? { ...qkNormRoPEFusion, headDim }
630
+ : null;
631
+ const canFuseSplitQKNormAndRoPE = qkNormRoPEFusion?.enabled === true
632
+ && qkNormRoPEFusion.projectionDiagnosticsEnabled !== true
633
+ && qkNormRoPEFusion.skipKNorm !== true
634
+ && qkNormRoPEFusion.allowUnitQKNorm !== true
635
+ && layerWeights.qNorm
636
+ && layerWeights.kNorm
637
+ && qkNormRoPEFusion.getNormWeightBuffer
638
+ && qkNormRoPEFusion.freqsCos
639
+ && qkNormRoPEFusion.freqsSin
640
+ && canUseSplitQKVRMSNormRoPEQK(qkvTensor, qkNormRoPEOptions);
641
+ if (canFuseSplitQKNormAndRoPE) {
642
+ qNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
643
+ kNormBuf = qkNormRoPEFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
644
+ const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
645
+ const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
646
+ if (qNormApplies && kNormApplies) {
647
+ const fused = await runSplitQKVRMSNormRoPEQKForMode(
648
+ qkvTensor,
649
+ qNormBuf,
650
+ kNormBuf,
651
+ qkNormRoPEFusion.freqsCos,
652
+ qkNormRoPEFusion.freqsSin,
653
+ qkNormRoPEFusion.rmsNormEps,
654
+ {
655
+ numTokens,
656
+ numHeads,
657
+ numKVHeads,
658
+ headDim,
659
+ qSize: qSizeFused,
660
+ kSize: kSizeFused,
661
+ vSize: vSizeFused,
662
+ startPos: qkNormRoPEFusion.startPos,
663
+ rotaryDim: qkNormRoPEFusion.rotaryDim,
664
+ pairSpanDim: qkNormRoPEFusion.pairSpanDim,
665
+ interleaved: qkNormRoPEFusion.interleaved,
666
+ rmsNormWeightOffset: qkNormRoPEFusion.rmsNormWeightOffset === true,
667
+ f16KVCacheWrite: qkNormRoPEFusion.f16KVCacheWrite ?? null,
668
+ }
669
+ );
670
+ releaseTemporary(qkvTensor.buffer);
671
+ qkvTensor = null;
672
+ if (onFusedQKV) {
673
+ onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
674
+ }
675
+ return {
676
+ qTensor: fused.Q,
677
+ qGateTensor: null,
678
+ kTensor: fused.K,
679
+ vTensor: fused.V,
680
+ usedFusedQKV: true,
681
+ valueAliasesKey: false,
682
+ qkNormApplied: true,
683
+ ropeApplied: true,
684
+ kvCacheWriteFused: fused.wroteF16KVCache === true,
685
+ };
686
+ }
687
+ }
688
+ const canFuseSplitAndQKNorm = qkNormFusion?.enabled === true
689
+ && qkNormFusion.projectionDiagnosticsEnabled !== true
690
+ && qkNormFusion.skipKNorm !== true
691
+ && qkNormFusion.allowUnitQKNorm !== true
692
+ && layerWeights.qNorm
693
+ && layerWeights.kNorm
694
+ && qkNormFusion.getNormWeightBuffer
695
+ && canUseSplitQKVRMSNormQK(qkvTensor, qkNormFusion);
696
+ if (canFuseSplitAndQKNorm) {
697
+ qNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
698
+ kNormBuf = qkNormFusion.getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
699
+ const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
700
+ const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
701
+ if (qNormApplies && kNormApplies) {
702
+ const fused = await runSplitQKVRMSNormQKForMode(
703
+ qkvTensor,
704
+ qNormBuf,
705
+ kNormBuf,
706
+ qkNormFusion.rmsNormEps,
707
+ {
708
+ numTokens,
709
+ numHeads,
710
+ numKVHeads,
711
+ headDim,
712
+ qSize: qSizeFused,
713
+ kSize: kSizeFused,
714
+ vSize: vSizeFused,
715
+ rmsNormWeightOffset: qkNormFusion.rmsNormWeightOffset === true,
716
+ }
717
+ );
718
+ releaseTemporary(qkvTensor.buffer);
719
+ qkvTensor = null;
720
+ if (onFusedQKV) {
721
+ onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
722
+ }
723
+ return {
724
+ qTensor: fused.Q,
725
+ qGateTensor: null,
726
+ kTensor: fused.K,
727
+ vTensor: fused.V,
728
+ usedFusedQKV: true,
729
+ valueAliasesKey: false,
730
+ qkNormApplied: true,
731
+ ropeApplied: false,
732
+ kvCacheWriteFused: false,
733
+ };
734
+ }
735
+ }
510
736
  const split = await runSplitForMode(qkvTensor, {
511
737
  numTokens,
512
738
  qSize: qSizeFused,
@@ -524,6 +750,9 @@ export async function projectAttentionQKV({
524
750
  vTensor: split.V,
525
751
  usedFusedQKV: true,
526
752
  valueAliasesKey: false,
753
+ qkNormApplied: false,
754
+ ropeApplied: false,
755
+ kvCacheWriteFused: false,
527
756
  };
528
757
  } catch (error) {
529
758
  if (qkvTensor) {
@@ -531,6 +760,8 @@ export async function projectAttentionQKV({
531
760
  }
532
761
  throw error;
533
762
  } finally {
763
+ releaseOwnedNormBuffer(qNormBuf, ownsNormBuffer(layerWeights.qNorm), releaseTemporary, releasedNormBuffers);
764
+ releaseOwnedNormBuffer(kNormBuf, ownsNormBuffer(layerWeights.kNorm), releaseTemporary, releasedNormBuffers);
534
765
  if (projectionInputOwned) {
535
766
  releaseTemporary(projectionInput.buffer);
536
767
  }
@@ -572,6 +803,9 @@ export async function projectAttentionQKV({
572
803
  vTensor: sharedVTensor,
573
804
  usedFusedQKV: false,
574
805
  valueAliasesKey: false,
806
+ qkNormApplied: false,
807
+ ropeApplied: false,
808
+ kvCacheWriteFused: false,
575
809
  };
576
810
  }
577
811
 
@@ -629,7 +863,17 @@ export async function projectAttentionQKV({
629
863
  valueAliasesKey = true;
630
864
  }
631
865
 
632
- return { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV: false, valueAliasesKey };
866
+ return {
867
+ qTensor,
868
+ qGateTensor,
869
+ kTensor,
870
+ vTensor,
871
+ usedFusedQKV: false,
872
+ valueAliasesKey,
873
+ qkNormApplied: false,
874
+ ropeApplied: false,
875
+ kvCacheWriteFused: false,
876
+ };
633
877
  } catch (error) {
634
878
  for (const tensor of [qTensor, qGateTensor]) {
635
879
  if (tensor?.buffer) {
@@ -690,21 +934,66 @@ export async function applyAttentionQKNorm({
690
934
  onKNormApplied = null,
691
935
  skipKNorm = false,
692
936
  retainKInput = false,
937
+ allowUnitQKNorm = false,
693
938
  }) {
694
939
  const runRmsNormForMode = getRmsNormRunner(recorder);
940
+ const runRmsNormQKForMode = getRmsNormQKRunner(recorder);
695
941
  let nextQ = qTensor;
696
942
  let nextK = kTensor;
943
+ let qNormBuf = null;
944
+ let kNormBuf = null;
945
+ const releasedNormBuffers = new Set();
946
+
947
+ try {
948
+ const wantsQNorm = (layerWeights.qNorm && getNormWeightBuffer) || allowUnitQKNorm;
949
+ const wantsKNorm = !skipKNorm && ((layerWeights.kNorm && getNormWeightBuffer) || allowUnitQKNorm);
697
950
 
698
- if (layerWeights.qNorm && getNormWeightBuffer) {
699
- const qNormBuf = getNormWeightBuffer(layerWeights.qNorm, 'q_norm');
700
- const qElemsF32 = qNormBuf.size / 4;
701
- const qElemsF16 = qNormBuf.size / 2;
702
- const qElems = qElemsF32 === headDim ? qElemsF32 : qElemsF16;
703
- if (qElems === headDim) {
951
+ if (wantsQNorm) {
952
+ qNormBuf = layerWeights.qNorm && getNormWeightBuffer
953
+ ? getNormWeightBuffer(layerWeights.qNorm, 'q_norm')
954
+ : getQKNormOnesBuffer(headDim);
955
+ }
956
+ if (wantsKNorm) {
957
+ kNormBuf = layerWeights.kNorm && getNormWeightBuffer
958
+ ? getNormWeightBuffer(layerWeights.kNorm, 'k_norm')
959
+ : getQKNormOnesBuffer(headDim);
960
+ }
961
+
962
+ const qNormApplies = normBufferMatchesHeadDim(qNormBuf, headDim);
963
+ const kNormApplies = normBufferMatchesHeadDim(kNormBuf, headDim);
964
+ if (
965
+ qNormApplies
966
+ && kNormApplies
967
+ && canUseRMSNormQK(nextQ, nextK, { skipKNorm })
968
+ ) {
969
+ const fused = await runRmsNormQKForMode(nextQ, nextK, qNormBuf, kNormBuf, rmsNormEps, {
970
+ numTokens,
971
+ numHeads,
972
+ numKVHeads,
973
+ headDim,
974
+ rmsNormWeightOffset,
975
+ });
976
+ releaseTemporary(nextQ.buffer);
977
+ if (!retainKInput) {
978
+ releaseTemporary(nextK.buffer);
979
+ }
980
+ nextQ = fused.q;
981
+ nextK = fused.k;
982
+ if (onQNormApplied) {
983
+ await onQNormApplied(nextQ);
984
+ }
985
+ if (onKNormApplied) {
986
+ await onKNormApplied(nextK);
987
+ }
988
+ return { qTensor: nextQ, kTensor: nextK };
989
+ }
990
+
991
+ if (qNormApplies) {
704
992
  const qNormedTensor = await runRmsNormForMode(nextQ, qNormBuf, rmsNormEps, {
705
993
  batchSize: numTokens * numHeads,
706
994
  hiddenSize: headDim,
707
995
  rmsNormWeightOffset,
996
+ label: 'q_norm',
708
997
  });
709
998
  releaseTemporary(nextQ.buffer);
710
999
  nextQ = qNormedTensor;
@@ -712,21 +1001,13 @@ export async function applyAttentionQKNorm({
712
1001
  await onQNormApplied(nextQ);
713
1002
  }
714
1003
  }
715
- if (!isGpuBufferInstance(layerWeights.qNorm) && !isWeightBuffer(layerWeights.qNorm)) {
716
- releaseTemporary(qNormBuf);
717
- }
718
- }
719
1004
 
720
- if (!skipKNorm && layerWeights.kNorm && getNormWeightBuffer) {
721
- const kNormBuf = getNormWeightBuffer(layerWeights.kNorm, 'k_norm');
722
- const kElemsF32 = kNormBuf.size / 4;
723
- const kElemsF16 = kNormBuf.size / 2;
724
- const kElems = kElemsF32 === headDim ? kElemsF32 : kElemsF16;
725
- if (kElems === headDim) {
1005
+ if (kNormApplies) {
726
1006
  const kNormedTensor = await runRmsNormForMode(nextK, kNormBuf, rmsNormEps, {
727
1007
  batchSize: numTokens * numKVHeads,
728
1008
  hiddenSize: headDim,
729
1009
  rmsNormWeightOffset,
1010
+ label: 'k_norm',
730
1011
  });
731
1012
  if (!retainKInput) {
732
1013
  releaseTemporary(nextK.buffer);
@@ -736,10 +1017,9 @@ export async function applyAttentionQKNorm({
736
1017
  await onKNormApplied(nextK);
737
1018
  }
738
1019
  }
739
- if (!isGpuBufferInstance(layerWeights.kNorm) && !isWeightBuffer(layerWeights.kNorm)) {
740
- releaseTemporary(kNormBuf);
741
- }
1020
+ return { qTensor: nextQ, kTensor: nextK };
1021
+ } finally {
1022
+ releaseOwnedNormBuffer(qNormBuf, ownsNormBuffer(layerWeights?.qNorm), releaseTemporary, releasedNormBuffers);
1023
+ releaseOwnedNormBuffer(kNormBuf, ownsNormBuffer(layerWeights?.kNorm), releaseTemporary, releasedNormBuffers);
742
1024
  }
743
-
744
- return { qTensor: nextQ, kTensor: nextK };
745
1025
  }
@@ -0,0 +1,38 @@
1
+ export interface QuantAttentionRuntimeState {
2
+ coldLen?: number;
3
+ hotLen?: number;
4
+ hotWindow?: number;
5
+ hotStart?: number;
6
+ coldPackedStride?: number;
7
+ coldQuantMode?: string;
8
+ prodMode?: boolean;
9
+ rotationMatrixBuffer?: GPUBuffer | null;
10
+ codebookCentroidsBuffer?: GPUBuffer | null;
11
+ residualKGPU?: GPUBuffer | null;
12
+ residualVGPU?: GPUBuffer | null;
13
+ residualNormsKGPU?: GPUBuffer | null;
14
+ residualNormsVGPU?: GPUBuffer | null;
15
+ qjlMatrixBuffer?: GPUBuffer | null;
16
+ residualPackedStride?: number;
17
+ }
18
+
19
+ export interface QuantAttentionInputOptions {
20
+ seqLen?: number;
21
+ kvLen?: number;
22
+ numKVHeads?: number;
23
+ causal?: boolean;
24
+ startPos?: number;
25
+ slidingWindow?: number;
26
+ attnSoftcap?: number;
27
+ scale?: number;
28
+ }
29
+
30
+ export declare function buildTieredQuantAttentionOptions(
31
+ kvState: QuantAttentionRuntimeState | null | undefined,
32
+ options?: QuantAttentionInputOptions
33
+ ): Record<string, unknown>;
34
+
35
+ export declare function buildContiguousQuantAttentionOptions(
36
+ kvState: QuantAttentionRuntimeState | null | undefined,
37
+ options?: QuantAttentionInputOptions
38
+ ): Record<string, unknown>;