doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -6,12 +6,10 @@ import { createTensor } from '../tensor.js';
6
6
  import { KernelBase } from './kernel-base.js';
7
7
  import { TILE_SIZES } from './constants.js';
8
8
  import { getKernelThresholds, padToQ4KBlock } from '../../config/schema/index.js';
9
- import {
10
- createUniformBufferWithView,
11
- getKernelConfig,
12
- getPipelineBindGroupLayout,
13
- hasRequiredFeatures,
14
- } from './utils.js';
9
+ import { createUniformBufferWithView } from './uniform-utils.js';
10
+ import { getKernelConfig } from './kernel-configs.js';
11
+ import { getPipelineBindGroupLayout } from './pipeline-cache.js';
12
+ import { hasRequiredFeatures } from './feature-check.js';
15
13
  import { dispatchIndirect, recordDispatchIndirect } from './dispatch.js';
16
14
  import { releaseUniformBuffer } from '../uniform-cache.js';
17
15
  import { log, trace } from '../../debug/index.js';
@@ -20,22 +18,10 @@ import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
20
18
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
21
19
  import { logKernelSelectionOnce } from '../kernel-selection-log.js';
22
20
  import { getRuntimeConfig } from '../../config/runtime.js';
21
+ import { getRequiredVariantMaxKVLen, resolveAttentionPlan } from './attention/plan.js';
22
+ import { assertAttentionBindGroupBuffer, dispatchAttentionKernel, executeAttention, executeAttentionBDPA, getPageTableFallbackBuffer, releaseAttentionUniform, resolveAttentionExecution } from './attention/executors.js';
23
23
 
24
24
  // Track if we've logged the attention tier selection (avoid spam)
25
- let loggedAttentionTier = false;
26
-
27
- function getRequiredVariantMaxKVLen(operation, variant, errorLabel) {
28
- const config = getKernelConfig(operation, variant);
29
- const maxKVLen = config.variantMetadata?.maxKVLen;
30
- if (!Number.isFinite(maxKVLen)) {
31
- throw new Error(`Kernel config missing ${errorLabel} maxKVLen`);
32
- }
33
- return maxKVLen;
34
- }
35
-
36
- function getChunkedMaxKVLen() {
37
- return getRequiredVariantMaxKVLen('attention', 'decode_chunked_f16kv', 'attention.decode_chunked_f16kv');
38
- }
39
25
 
40
26
  function getTieredMaxKVLen() {
41
27
  return getRequiredVariantMaxKVLen('attention_tiered', 'decode_tiered_f16', 'attention_tiered.decode_tiered_f16');
@@ -57,107 +43,12 @@ function getContiguousQuantMaxKVLen() {
57
43
  );
58
44
  }
59
45
 
60
-
61
- let kvLenFallbackBuffer = null;
62
- let kvLenFallbackBufferEpoch = -1;
63
- const U32_BYTES = Uint32Array.BYTES_PER_ELEMENT;
64
- const F32_BYTES = Float32Array.BYTES_PER_ELEMENT;
65
-
66
-
67
- function getKvLenFallbackBuffer(device) {
68
- const epoch = getDeviceEpoch();
69
- if (!kvLenFallbackBuffer || kvLenFallbackBufferEpoch !== epoch) {
70
- kvLenFallbackBuffer = device.createBuffer({
71
- label: 'attention_kv_len_fallback',
72
- size: U32_BYTES,
73
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
74
- });
75
- device.queue.writeBuffer(kvLenFallbackBuffer, 0, new Uint32Array([0]));
76
- kvLenFallbackBufferEpoch = epoch;
77
- }
78
- return kvLenFallbackBuffer;
79
- }
80
-
81
- let pageTableFallbackBuffer = null;
82
- let pageTableFallbackBufferEpoch = -1;
83
-
84
- function getPageTableFallbackBuffer(device) {
85
- const epoch = getDeviceEpoch();
86
- if (!pageTableFallbackBuffer || pageTableFallbackBufferEpoch !== epoch) {
87
- pageTableFallbackBuffer = device.createBuffer({
88
- label: 'attention_page_table_fallback',
89
- size: U32_BYTES,
90
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
91
- });
92
- device.queue.writeBuffer(pageTableFallbackBuffer, 0, new Uint32Array([0]));
93
- pageTableFallbackBufferEpoch = epoch;
94
- }
95
- return pageTableFallbackBuffer;
96
- }
97
-
98
- class AttentionKernel extends KernelBase {
99
-
100
- async getPipeline(variant, constants = null) {
101
- return this.getPipelineFor('attention', variant, null, constants);
102
- }
103
-
104
-
105
- dispatch(
106
- pipeline,
107
- bindGroup,
108
- workgroups
109
- ) {
110
- this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention');
111
- }
112
-
113
-
114
- record(
115
- recorder,
116
- pipeline,
117
- bindGroup,
118
- workgroups
119
- ) {
120
- this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention');
121
- }
122
- }
123
-
124
- function resolveOnlineDecodePipelineConstants(variant, headDim, options = {}) {
125
- if (!variant?.startsWith('decode_online')) {
126
- return null;
127
- }
128
- const policy = getRuntimeConfig()?.inference?.session?.attentionDecodeOnline;
129
- const workgroupSize = policy?.workgroupSize;
130
- const constants = {};
131
- if (workgroupSize != null) {
132
- if (workgroupSize !== 128 && workgroupSize !== 256) {
133
- throw new Error(`attentionDecodeOnline.workgroupSize must be 128 or 256, got ${workgroupSize}.`);
134
- }
135
- if (headDim > workgroupSize * 2) {
136
- throw new Error(
137
- `attentionDecodeOnline.workgroupSize=${workgroupSize} cannot cover headDim=${headDim}; ` +
138
- `requires headDim <= ${workgroupSize * 2}.`
139
- );
140
- }
141
- constants.WORKGROUP_SIZE = workgroupSize;
142
- }
143
- if (
144
- variant === 'decode_online_head256_f16kv_output_gate'
145
- && policy?.useDirectContiguousKVLayout === true
146
- && options.kvLayout === 'contiguous'
147
- && options.slidingWindow === 0
148
- ) {
149
- constants.USE_DIRECT_KV_LAYOUT = true;
150
- }
151
- return Object.keys(constants).length > 0 ? constants : null;
152
- }
153
-
154
46
  class AttentionTieredKernel extends KernelBase {
155
47
 
156
48
  async getPipeline(variant) {
157
49
  return this.getPipelineFor('attention_tiered', variant);
158
50
  }
159
51
 
160
-
161
52
  dispatch(
162
53
  pipeline,
163
54
  bindGroup,
@@ -166,7 +57,6 @@ class AttentionTieredKernel extends KernelBase {
166
57
  this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_tiered');
167
58
  }
168
59
 
169
-
170
60
  record(
171
61
  recorder,
172
62
  pipeline,
@@ -183,1478 +73,189 @@ class AttentionTieredQuantKernel extends KernelBase {
183
73
  return this.getPipelineFor('attention_tiered_quant', variant);
184
74
  }
185
75
 
186
-
187
- dispatch(
188
- pipeline,
189
- bindGroup,
190
- workgroups
191
- ) {
192
- this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_tiered_quant');
193
- }
194
-
195
-
196
- record(
197
- recorder,
198
- pipeline,
199
- bindGroup,
200
- workgroups
201
- ) {
202
- this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_tiered_quant');
203
- }
204
- }
205
-
206
- class AttentionContiguousQuantKernel extends KernelBase {
207
-
208
- async getPipeline(variant) {
209
- return this.getPipelineFor('attention_contiguous_quant', variant);
210
- }
211
-
212
- dispatch(
213
- pipeline,
214
- bindGroup,
215
- workgroups
216
- ) {
217
- this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
218
- }
219
-
220
- record(
221
- recorder,
222
- pipeline,
223
- bindGroup,
224
- workgroups
225
- ) {
226
- this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
227
- }
228
- }
229
-
230
- class AttentionBDPAKernel extends KernelBase {
231
- async getPipeline(variant) {
232
- return this.getPipelineFor('attention_bdpa', variant);
233
- }
234
-
235
76
  dispatch(
236
77
  pipeline,
237
78
  bindGroup,
238
- workgroups
239
- ) {
240
- this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_bdpa');
241
- }
242
-
243
- record(
244
- recorder,
245
- pipeline,
246
- bindGroup,
247
- workgroups
248
- ) {
249
- this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_bdpa');
250
- }
251
- }
252
-
253
- function selectAttentionTier(
254
- headDim,
255
- seqLen,
256
- useF16KV,
257
- forcedTier,
258
- sharedLimit,
259
- caps
260
- ) {
261
- const isDecode = seqLen === 1;
262
- const thresholds = getKernelThresholds().attention;
263
- const largeRequired = useF16KV
264
- ? thresholds.largeSharedF16
265
- : thresholds.largeSharedF32;
266
- const canLarge =
267
- headDim <= thresholds.largeMaxHeadDim &&
268
- sharedLimit >= largeRequired;
269
- const smallRequired = useF16KV
270
- ? thresholds.smallSharedF16
271
- : thresholds.smallSharedF32;
272
- const canSmall =
273
- headDim <= thresholds.smallMaxHeadDim &&
274
- sharedLimit >= smallRequired;
275
- const canSubgroup =
276
- caps.hasSubgroups &&
277
- headDim <= thresholds.subgroupMaxHeadDim &&
278
- sharedLimit >= thresholds.subgroupShared &&
279
- isDecode;
280
-
281
-
282
- let tier = forcedTier;
283
- let reason = forcedTier ? `forced:${forcedTier}` : '';
284
-
285
- if (tier === 'tiled_large' && !canLarge) {
286
- throw new Error(`Requested tiled_large but device doesn't support it (headDim=${headDim}, shared=${sharedLimit}).`);
287
- }
288
- if (tier === 'tiled_small' && !canSmall) {
289
- throw new Error(`Requested tiled_small but device doesn't support it (headDim=${headDim}, shared=${sharedLimit}).`);
290
- }
291
- if (tier === 'subgroup' && !canSubgroup) {
292
- throw new Error(`Requested subgroup attention but device doesn't support it (headDim=${headDim}, shared=${sharedLimit}, subgroups=${caps.hasSubgroups}).`);
293
- }
294
-
295
- if (!tier) {
296
- tier = selectKernelRuleValue('attention', 'tier', { canSubgroup, canLarge, canSmall, isDecode });
297
- if (!reason) {
298
- if (canSubgroup) {
299
- reason = 'subgroup_capable';
300
- } else if (canLarge) {
301
- reason = 'tiled_large_capable';
302
- } else if (canSmall) {
303
- reason = 'tiled_small_capable';
304
- } else if (isDecode) {
305
- reason = 'decode_streaming_fallback';
306
- } else {
307
- reason = 'streaming_fallback';
308
- }
309
- }
310
- if (tier === 'subgroup' && !loggedAttentionTier) {
311
- trace.attn(0, `Using subgroup decode kernel (headDim=${headDim}, hasSubgroups=true)`);
312
- loggedAttentionTier = true;
313
- }
314
- }
315
-
316
- return { tier, reason };
317
- }
318
-
319
- // Track if we've logged chunked kernel selection
320
- let loggedChunkedKernel = false;
321
-
322
-
323
- function resolveAttentionVariant(
324
- tier,
325
- isDecode,
326
- useF16KV,
327
- useF16Q,
328
- numHeads,
329
- headDim,
330
- kvLen,
331
- isPaged,
332
- caps,
333
- sharedLimit
334
- ) {
335
- const base = selectKernelRuleValue('attention', 'phase', { isDecode });
336
- const useF16 = useF16KV && useF16Q;
337
- const suffix = selectKernelRuleValue('attention', 'suffix', { useF16, useF16KV });
338
-
339
- // Check if chunked kernel is viable:
340
- // - Decode only (seqLen=1)
341
- // - F16 KV cache
342
- // - Large headDim (parallelizes across dimensions)
343
- // - KV length within shared memory limit (from kernel config)
344
- const chunkedMaxKVLen = getChunkedMaxKVLen();
345
- const minHeadDimForChunked = getKernelThresholds().attention.minHeadDimForChunked;
346
- const canUseChunked = isDecode && useF16KV && headDim >= minHeadDimForChunked && kvLen <= chunkedMaxKVLen;
347
- const decodeSubgroupMaxKVLen = chunkedMaxKVLen;
348
- const decodeSubgroupMaxHeadDim = getKernelThresholds().attention.subgroupMaxHeadDim;
349
- const canUseDecodeSubgroup = isDecode && !useF16KV && !useF16Q && headDim <= decodeSubgroupMaxHeadDim && kvLen <= decodeSubgroupMaxKVLen;
350
- const canUseDecodeOptimized = isDecode
351
- && useF16KV
352
- && caps.hasF16
353
- && caps.hasSubgroups
354
- && headDim <= decodeSubgroupMaxHeadDim
355
- && sharedLimit >= getKernelThresholds().attention.subgroupShared;
356
- const chunkedVariant = selectKernelRuleValue('attention', 'chunkedVariant', { useF16 });
357
- const pagedVariant = selectKernelRuleValue('attention', 'pagedVariant', { useF16 });
358
- const optimizedVariant = selectKernelRuleValue('attention', 'optimizedVariant', { useF16 });
359
- const variant = selectKernelRuleValue(
360
- 'attention',
361
- 'variant',
362
- {
363
- tier,
364
- useF16KV,
365
- canUseChunked,
366
- canUseDecodeSubgroup,
367
- canUseDecodeOptimized,
368
- base,
369
- suffix,
370
- chunkedVariant,
371
- pagedVariant,
372
- optimizedVariant,
373
- isPaged,
374
- isDecode,
375
- }
376
- );
377
-
378
- if (variant === chunkedVariant && !loggedChunkedKernel) {
379
- trace.attn(0, `Using chunked decode kernel (headDim=${headDim}, numHeads=${numHeads}, f16kv=${!useF16Q})`);
380
- loggedChunkedKernel = true;
381
- }
382
-
383
- return variant;
384
- }
385
-
386
-
387
- function resolveAttentionQueryBlockSize(tier, variant = null) {
388
- if (variant) {
389
- const metadataBlockSize = getKernelConfig('attention', variant).variantMetadata?.queryBlockSize;
390
- if (metadataBlockSize != null) {
391
- if (!Number.isInteger(metadataBlockSize) || metadataBlockSize <= 0) {
392
- throw new Error(`Attention kernel "${variant}" has invalid variantMetadata.queryBlockSize=${metadataBlockSize}.`);
393
- }
394
- return metadataBlockSize;
395
- }
396
- }
397
- if (tier === 'tiled_large') {
398
- return TILE_SIZES.ATTENTION_LARGE_BLOCK_SIZE;
399
- }
400
- return TILE_SIZES.ATTENTION_SMALL_BLOCK_SIZE;
401
- }
402
-
403
- function calculateAttentionWorkgroups(tier, seqLen, numHeads, variant = null) {
404
- if (tier === 'subgroup') {
405
- return numHeads;
406
- }
407
- if (tier === 'streaming') {
408
- return seqLen * numHeads;
409
- }
410
- const queryBlockSize = resolveAttentionQueryBlockSize(tier, variant);
411
- return Math.ceil(seqLen / queryBlockSize) * numHeads;
412
- }
413
-
414
-
415
- function inferAttentionTierFromVariant(variant) {
416
- const config = getKernelConfig('attention', variant);
417
- const tier = config.variantMetadata?.tier;
418
- if (!tier) {
419
- throw new Error(`Attention kernel "${variant}" missing variantMetadata.tier in registry.`);
420
- }
421
- return tier;
422
- }
423
-
424
-
425
- function validateAttentionVariant(
426
- variant,
427
- isDecode,
428
- useF16KV,
429
- useF16Q,
430
- caps,
431
- headDim,
432
- kvLen,
433
- sharedLimit
434
- ) {
435
- const normalized = variant.trim();
436
-
437
- let config;
438
- try {
439
- config = getKernelConfig('attention', normalized);
440
- } catch {
441
- throw new Error(`Unknown attention kernel variant "${variant}".`);
442
- }
443
-
444
- if (!hasRequiredFeatures(config.requires, caps)) {
445
- throw new Error(`Attention kernel "${variant}" requires unsupported GPU features.`);
446
- }
447
-
448
- const expectsF16KV = normalized.includes('_f16kv');
449
- const expectsF16 = normalized.includes('_f16') && !expectsF16KV;
450
- if (expectsF16) {
451
- if (!(useF16KV && useF16Q)) {
452
- const kvLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16KV });
453
- const qLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16Q });
454
- throw new Error(`Attention kernel "${variant}" requires f16 Q/K/V but got Q=${qLabel}, KV=${kvLabel}.`);
455
- }
456
- } else if (expectsF16KV) {
457
- if (!useF16KV || useF16Q) {
458
- const kvLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16KV });
459
- const qLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16Q });
460
- throw new Error(`Attention kernel "${variant}" requires f32 Q with f16 KV but got Q=${qLabel}, KV=${kvLabel}.`);
461
- }
462
- } else {
463
- if (useF16KV || useF16Q) {
464
- const kvLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16KV });
465
- const qLabel = selectSharedRuleValue('shared', 'dtype', 'f16OrF32', { useF16: useF16Q });
466
- throw new Error(`Attention kernel "${variant}" requires f32 Q/K/V but got Q=${qLabel}, KV=${kvLabel}.`);
467
- }
468
- }
469
-
470
- const isDecodeVariant = normalized.startsWith('decode');
471
- const isPrefillVariant = normalized.startsWith('prefill');
472
- if (isDecode && isPrefillVariant) {
473
- throw new Error(`Attention kernel "${variant}" is prefill-only but decode requested.`);
474
- }
475
- if (!isDecode && isDecodeVariant) {
476
- throw new Error(`Attention kernel "${variant}" is decode-only but prefill requested.`);
477
- }
478
-
479
- const thresholds = getKernelThresholds().attention;
480
- const chunkedMaxKVLen = getChunkedMaxKVLen();
481
- const isChunked = normalized.startsWith('decode_chunked');
482
- if (isChunked) {
483
- const minHeadDimForChunked = thresholds.minHeadDimForChunked;
484
- if (headDim < minHeadDimForChunked) {
485
- throw new Error(`Attention kernel "${variant}" requires headDim >= ${minHeadDimForChunked} but got ${headDim}.`);
486
- }
487
- if (kvLen > chunkedMaxKVLen) {
488
- throw new Error(`Attention kernel "${variant}" requires kvLen <= ${chunkedMaxKVLen} but got ${kvLen}.`);
489
- }
490
- }
491
-
492
- if (normalized === 'decode_subgroup') {
493
- if (!caps.hasSubgroups) {
494
- throw new Error(`Attention kernel "${variant}" requires subgroup support.`);
495
- }
496
- if (headDim > thresholds.subgroupMaxHeadDim) {
497
- throw new Error(`Attention kernel "${variant}" requires headDim <= ${thresholds.subgroupMaxHeadDim} but got ${headDim}.`);
498
- }
499
- if (kvLen > chunkedMaxKVLen) {
500
- throw new Error(`Attention kernel "${variant}" requires kvLen <= ${chunkedMaxKVLen} but got ${kvLen}.`);
501
- }
502
- if (sharedLimit < thresholds.subgroupShared) {
503
- throw new Error(`Attention kernel "${variant}" requires shared >= ${thresholds.subgroupShared} but got ${sharedLimit}.`);
504
- }
505
- }
506
-
507
- if (normalized.startsWith('decode_online')) {
508
- const exactHeadDim = config.variantMetadata?.exactHeadDim;
509
- const maxHeadDim = config.variantMetadata?.maxHeadDim ?? thresholds.subgroupMaxHeadDim;
510
- if (!caps.hasSubgroups) {
511
- throw new Error(`Attention kernel "${variant}" requires subgroup support.`);
512
- }
513
- if (Number.isFinite(exactHeadDim) && headDim !== exactHeadDim) {
514
- throw new Error(`Attention kernel "${variant}" requires headDim == ${exactHeadDim} but got ${headDim}.`);
515
- }
516
- if (headDim > maxHeadDim) {
517
- throw new Error(`Attention kernel "${variant}" requires headDim <= ${maxHeadDim} but got ${headDim}.`);
518
- }
519
- if (sharedLimit < thresholds.subgroupShared) {
520
- throw new Error(`Attention kernel "${variant}" requires shared >= ${thresholds.subgroupShared} but got ${sharedLimit}.`);
521
- }
522
- }
523
-
524
- if (normalized.startsWith('prefill') || normalized.startsWith('decode')) {
525
- const isSmall = normalized.includes('_small');
526
- const isStreaming = normalized.includes('_streaming');
527
- const isTiled = !isStreaming
528
- && !normalized.startsWith('decode_subgroup')
529
- && !normalized.startsWith('decode_online')
530
- && !isChunked;
531
- if (isTiled) {
532
- const metadata = config.variantMetadata ?? {};
533
- const requiredShared = metadata.requiredShared ?? (
534
- isSmall
535
- ? (useF16KV ? thresholds.smallSharedF16 : thresholds.smallSharedF32)
536
- : (useF16KV ? thresholds.largeSharedF16 : thresholds.largeSharedF32)
537
- );
538
- const maxHeadDim = metadata.maxHeadDim ?? (isSmall ? thresholds.smallMaxHeadDim : thresholds.largeMaxHeadDim);
539
- const minHeadDim = metadata.minHeadDim ?? 0;
540
- const exactHeadDim = metadata.exactHeadDim;
541
- if (Number.isFinite(exactHeadDim) && headDim !== exactHeadDim) {
542
- throw new Error(`Attention kernel "${variant}" requires headDim == ${exactHeadDim} but got ${headDim}.`);
543
- }
544
- if (headDim < minHeadDim) {
545
- throw new Error(`Attention kernel "${variant}" requires headDim >= ${minHeadDim} but got ${headDim}.`);
546
- }
547
- if (headDim > maxHeadDim) {
548
- throw new Error(`Attention kernel "${variant}" requires headDim <= ${maxHeadDim} but got ${headDim}.`);
549
- }
550
- if (sharedLimit < requiredShared) {
551
- throw new Error(`Attention kernel "${variant}" requires shared >= ${requiredShared} but got ${sharedLimit}.`);
552
- }
553
- }
554
- }
555
-
556
- return normalized;
557
- }
558
-
559
-
560
- function resolveAttentionPlan(
561
- seqLen,
562
- kvLen,
563
- headDim,
564
- numHeads,
565
- kvDtype,
566
- qDtype,
567
- sharedLimit,
568
- caps,
569
- layerIdx,
570
- isPaged,
571
- kernelPath
572
- ) {
573
- const useF16KV = kvDtype === 'f16';
574
- const useF16Q = qDtype === 'f16';
575
- const isDecode = seqLen === 1;
576
- const phase = selectKernelRuleValue('attention', 'phase', { isDecode });
577
- const pathVariant = getKernelPathAttentionVariant(phase, layerIdx, kernelPath);
578
- const strictPath = getKernelPathStrict();
579
-
580
- if (pathVariant) {
581
- let variantOverride;
582
- try {
583
- variantOverride = validateAttentionVariant(
584
- pathVariant,
585
- isDecode,
586
- useF16KV,
587
- useF16Q,
588
- caps,
589
- headDim,
590
- kvLen,
591
- sharedLimit
592
- );
593
- } catch (error) {
594
- if (strictPath) {
595
- throw error;
596
- }
597
- const reason = error instanceof Error ? error.message : String(error);
598
- log.warn(
599
- 'Attention',
600
- `Kernel path override "${pathVariant}" rejected; falling back to capability selection: ${reason}`
601
- );
602
- const adaptiveSelection = selectAttentionTier(headDim, seqLen, useF16KV, null, sharedLimit, caps);
603
- const adaptiveVariant = resolveAttentionVariant(
604
- adaptiveSelection.tier,
605
- isDecode,
606
- useF16KV,
607
- useF16Q,
608
- numHeads,
609
- headDim,
610
- kvLen,
611
- isPaged,
612
- caps,
613
- sharedLimit
614
- );
615
- const workgroups = calculateAttentionWorkgroups(adaptiveSelection.tier, seqLen, numHeads, adaptiveVariant);
616
- logKernelSelectionOnce('attention', {
617
- variant: adaptiveVariant,
618
- reason: `path_override_fallback:${adaptiveSelection.tier}`,
619
- });
620
- return {
621
- tier: adaptiveSelection.tier,
622
- variant: adaptiveVariant,
623
- workgroups,
624
- useF16KV,
625
- isDecode,
626
- };
627
- }
628
- let selectionReason = 'path_override';
629
-
630
- if (!isDecode && variantOverride.startsWith('prefill_streaming') && seqLen <= 64) {
631
- const adaptivePrefillVariant = variantOverride.endsWith('_f16kv')
632
- ? 'prefill_f16kv'
633
- : variantOverride.endsWith('_f16')
634
- ? 'prefill_f16'
635
- : 'prefill';
636
- try {
637
- const validatedAdaptive = validateAttentionVariant(
638
- adaptivePrefillVariant,
639
- isDecode,
640
- useF16KV,
641
- useF16Q,
642
- caps,
643
- headDim,
644
- kvLen,
645
- sharedLimit
646
- );
647
- if (validatedAdaptive !== variantOverride) {
648
- variantOverride = validatedAdaptive;
649
- selectionReason = 'path_override_adaptive_prefill';
650
- }
651
- } catch {
652
- // Keep original strict-path variant when adaptive fallback is not valid.
653
- }
654
- }
655
-
656
- const tier = inferAttentionTierFromVariant(variantOverride);
657
- const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads, variantOverride);
658
- logKernelSelectionOnce('attention', {
659
- variant: variantOverride,
660
- reason: `${selectionReason}:${tier}`,
661
- });
662
- return { tier, variant: variantOverride, workgroups, useF16KV, isDecode };
663
- }
664
-
665
- const selection = selectAttentionTier(headDim, seqLen, useF16KV, null, sharedLimit, caps);
666
- const tier = selection.tier;
667
- const variant = resolveAttentionVariant(
668
- tier,
669
- isDecode,
670
- useF16KV,
671
- useF16Q,
672
- numHeads,
673
- headDim,
674
- kvLen,
675
- isPaged,
676
- caps,
677
- sharedLimit
678
- );
679
- const validatedVariant = validateAttentionVariant(
680
- variant,
681
- isDecode,
682
- useF16KV,
683
- useF16Q,
684
- caps,
685
- headDim,
686
- kvLen,
687
- sharedLimit
688
- );
689
- const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads, variant);
690
-
691
- logKernelSelectionOnce('attention', {
692
- variant: validatedVariant,
693
- reason: selection.reason,
694
- });
695
-
696
- return { tier, variant: validatedVariant, workgroups, useF16KV, isDecode };
697
- }
698
-
699
- export function resolveAttentionPlanForTest(
700
- seqLen,
701
- kvLen,
702
- headDim,
703
- numHeads,
704
- kvDtype,
705
- qDtype,
706
- sharedLimit,
707
- caps,
708
- layerIdx,
709
- isPaged = false,
710
- kernelPath = null
711
- ) {
712
- return resolveAttentionPlan(
713
- seqLen,
714
- kvLen,
715
- headDim,
716
- numHeads,
717
- kvDtype,
718
- qDtype,
719
- sharedLimit,
720
- caps,
721
- layerIdx,
722
- isPaged,
723
- kernelPath
724
- );
725
- }
726
-
727
-
728
- function createAttentionUniformBuffer(
729
- device,
730
- recorder,
731
- params
732
- ) {
733
- return createUniformBufferWithView(
734
- 'attention_uniforms',
735
- 80,
736
- (view) => {
737
- view.setUint32(0, params.numHeads, true);
738
- view.setUint32(4, params.numKVHeads, true);
739
- view.setUint32(8, params.headDim, true);
740
- view.setUint32(12, params.kvLen, true);
741
- view.setUint32(16, params.seqLen, true);
742
- view.setFloat32(20, params.scale, true);
743
- view.setUint32(24, params.causal ? 1 : 0, true);
744
- view.setUint32(28, params.startPos, true);
745
- view.setFloat32(32, params.attnSoftcap, true); // Gemma 2: 50.0, 0 = disabled
746
- view.setUint32(36, params.slidingWindow, true); // Sliding window size, 0 = disabled
747
- view.setUint32(40, params.kvLenSource, true); // 0 = uniform kvLen, 1 = buffer
748
- view.setUint32(44, params.kvStart ?? 0, true);
749
- view.setUint32(48, params.pageSize ?? 0, true);
750
- view.setUint32(52, params.kvLayout ?? 0, true);
751
- view.setUint32(56, params.bidirectionalSpanStart ?? 0, true);
752
- view.setUint32(60, params.bidirectionalSpanLength ?? 0, true);
753
- view.setUint32(64, 0, true);
754
- view.setUint32(68, 0, true);
755
- view.setUint32(72, 0, true);
756
- },
757
- recorder,
758
- device
759
- );
760
- }
761
-
762
- function createTieredAttentionUniformBuffer(
763
- device,
764
- recorder,
765
- params
766
- ) {
767
- return createUniformBufferWithView(
768
- 'attention_tiered_uniforms',
769
- 80,
770
- (view) => {
771
- view.setUint32(0, params.numHeads, true);
772
- view.setUint32(4, params.numKVHeads, true);
773
- view.setUint32(8, params.headDim, true);
774
- view.setUint32(12, params.coldLen, true);
775
- view.setUint32(16, params.hotLen, true);
776
- view.setUint32(20, params.seqLen, true);
777
- view.setFloat32(24, params.scale, true);
778
- view.setUint32(28, params.causal ? 1 : 0, true);
779
- view.setUint32(32, params.startPos, true);
780
- view.setFloat32(36, params.attnSoftcap, true);
781
- view.setUint32(40, params.slidingWindow, true);
782
- view.setUint32(44, params.hotWindow, true);
783
- view.setUint32(48, params.hotStart, true);
784
- view.setUint32(52, params.coldPageSize, true);
785
- view.setUint32(56, params.coldLayout ?? 0, true);
786
- view.setUint32(60, params.hotLayout ?? 1, true);
787
- view.setUint32(64, 0, true);
788
- },
789
- recorder,
790
- device
791
- );
792
- }
793
-
794
- function createTieredQuantAttentionUniformBuffer(
795
- device,
796
- recorder,
797
- params
798
- ) {
799
- return createUniformBufferWithView(
800
- 'attention_tiered_quant_uniforms',
801
- 64,
802
- (view) => {
803
- view.setUint32(0, params.numHeads, true);
804
- view.setUint32(4, params.numKVHeads, true);
805
- view.setUint32(8, params.headDim, true);
806
- view.setUint32(12, params.coldLen, true);
807
- view.setUint32(16, params.hotLen, true);
808
- view.setUint32(20, params.seqLen, true);
809
- view.setFloat32(24, params.scale, true);
810
- view.setUint32(28, params.causal ? 1 : 0, true);
811
- view.setUint32(32, params.startPos, true);
812
- view.setFloat32(36, params.attnSoftcap, true);
813
- view.setUint32(40, params.slidingWindow, true);
814
- view.setUint32(44, params.hotWindow, true);
815
- view.setUint32(48, params.hotStart, true);
816
- view.setUint32(52, params.packedStride, true);
817
- view.setUint32(56, 0, true);
818
- },
819
- recorder,
820
- device
821
- );
822
- }
823
-
824
- function createContiguousQuantAttentionUniformBuffer(
825
- device,
826
- recorder,
827
- params
828
- ) {
829
- const hasProdFields = params.packedStrideMSE != null;
830
- const size = hasProdFields ? 64 : 48;
831
- return createUniformBufferWithView(
832
- 'attention_contiguous_quant_uniforms',
833
- size,
834
- (view) => {
835
- view.setUint32(0, params.numHeads, true);
836
- view.setUint32(4, params.numKVHeads, true);
837
- view.setUint32(8, params.headDim, true);
838
- view.setUint32(12, params.kvLen, true);
839
- view.setUint32(16, params.seqLen, true);
840
- view.setFloat32(20, params.scale, true);
841
- view.setUint32(24, params.causal ? 1 : 0, true);
842
- view.setUint32(28, params.startPos, true);
843
- view.setFloat32(32, params.attnSoftcap, true);
844
- view.setUint32(36, params.slidingWindow, true);
845
- if (hasProdFields) {
846
- view.setUint32(40, params.packedStrideMSE, true);
847
- view.setUint32(44, params.packedStrideResidual, true);
848
- view.setUint32(48, 0, true);
849
- view.setUint32(52, 0, true);
850
- view.setUint32(56, 0, true);
851
- view.setUint32(60, 0, true);
852
- } else {
853
- view.setUint32(40, params.packedStride, true);
854
- view.setUint32(44, 0, true);
855
- }
856
- },
857
- recorder,
858
- device
859
- );
860
- }
861
-
862
- function createBDPAAttentionUniformBuffer(
863
- device,
864
- recorder,
865
- params
866
- ) {
867
- return createUniformBufferWithView(
868
- 'attention_bdpa_uniforms',
869
- 64,
870
- (view) => {
871
- view.setUint32(0, params.numHeads, true);
872
- view.setUint32(4, params.numKVHeads, true);
873
- view.setUint32(8, params.headDim, true);
874
- view.setUint32(12, params.kvLen, true);
875
- view.setUint32(16, params.seqLen, true);
876
- view.setFloat32(20, params.scale, true);
877
- view.setUint32(24, params.causal ? 1 : 0, true);
878
- view.setUint32(28, params.startPos, true);
879
- view.setFloat32(32, params.attnSoftcap, true);
880
- view.setUint32(36, params.slidingWindow, true);
881
- view.setUint32(40, 0, true); // padding
882
- view.setUint32(44, 0, true); // padding
883
- view.setUint32(48, 0, true); // padding
884
- view.setUint32(52, 0, true); // padding
885
- view.setUint32(56, 0, true); // padding
886
- view.setUint32(60, 0, true); // padding
887
- },
888
- recorder,
889
- device
890
- );
891
- }
892
-
893
- function resolveAttentionExecution(recorder) {
894
- return {
895
- recorder: recorder || null,
896
- device: recorder?.device || getDevice(),
897
- };
898
- }
899
-
900
- function assertAttentionBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel, buffer, details = []) {
901
- const isGpuBuffer = buffer && (
902
- typeof GPUBuffer === 'undefined'
903
- ? true
904
- : buffer instanceof GPUBuffer
905
- );
906
- if (isGpuBuffer) {
907
- return;
908
- }
909
- const detailText = details.filter(Boolean).join(', ');
910
- throw new Error(
911
- `[${kernelName}] variant="${variant}" binding ${bindingIndex} "${bindingLabel}" requires a GPUBuffer` +
912
- (detailText ? ` (${detailText})` : '') +
913
- '.'
914
- );
915
- }
916
-
917
- function releaseAttentionUniform(execution, uniformBuffer) {
918
- if (!execution.recorder) {
919
- releaseUniformBuffer(uniformBuffer);
920
- }
921
- }
922
-
923
- function dispatchAttentionKernel(execution, kernel, pipeline, bindGroup, workgroups) {
924
- if (execution.recorder) {
925
- kernel.record(execution.recorder, pipeline, bindGroup, workgroups);
926
- return;
927
- }
928
- kernel.dispatch(pipeline, bindGroup, workgroups);
929
- }
930
-
931
- async function executeAttentionBDPA(
932
- recorder,
933
- Q,
934
- basisK,
935
- basisV,
936
- pagedK,
937
- pagedV,
938
- index,
939
- numHeads,
940
- headDim,
941
- options = {}
942
- ) {
943
- const execution = resolveAttentionExecution(recorder);
944
- const {
945
- seqLen = 1,
946
- kvLen = seqLen,
947
- numKVHeads = numHeads,
948
- scale = 1.0 / Math.sqrt(headDim),
949
- causal = true,
950
- startPos = 0,
951
- outputBuffer = null,
952
- attnSoftcap = 0,
953
- slidingWindow = 0,
954
- ropeCos = null,
955
- ropeSin = null,
956
- } = options;
957
-
958
- if (seqLen !== 1) {
959
- throw new Error(`BDPA attention currently supports decode only (seqLen=1), got seqLen=${seqLen}.`);
960
- }
961
- if (Q.dtype !== 'f16' || basisK.dtype !== 'f16' || basisV.dtype !== 'f16') {
962
- throw new Error(`BDPA attention requires f16 Q/basis tensors; got Q=${Q.dtype}, basisK=${basisK.dtype}, basisV=${basisV.dtype}.`);
963
- }
964
- if (!(ropeCos instanceof GPUBuffer) || !(ropeSin instanceof GPUBuffer)) {
965
- throw new Error('BDPA attention requires GPU ropeCos/ropeSin buffers.');
966
- }
967
-
968
- const variant = 'decode_bdpa_f16';
969
- const caps = getKernelCapabilities();
970
- const config = getKernelConfig('attention_bdpa', variant);
971
- if (!hasRequiredFeatures(config.requires, caps)) {
972
- throw new Error(`BDPA attention kernel "${variant}" requires unsupported GPU features.`);
973
- }
974
- const maxKVLen = config.variantMetadata?.maxKVLen;
975
- if (Number.isFinite(maxKVLen) && kvLen > maxKVLen) {
976
- throw new Error(`BDPA attention requires kvLen <= ${maxKVLen} but got ${kvLen}.`);
977
- }
978
-
979
- const kernel = new AttentionBDPAKernel(execution.device);
980
- const pipeline = await kernel.getPipeline(variant);
981
-
982
- const outputDtype = config.outputDtype;
983
- if (!outputDtype) {
984
- throw new Error(`Kernel config missing outputDtype for attention_bdpa variant "${variant}".`);
985
- }
986
- const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
987
- const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
988
- const outputSize = seqLen * paddedHiddenSize * bytesPerElement;
989
- const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_bdpa_output');
990
-
991
- const uniformBuffer = createBDPAAttentionUniformBuffer(execution.device, execution.recorder, {
992
- numHeads,
993
- numKVHeads,
994
- headDim,
995
- kvLen,
996
- seqLen,
997
- scale,
998
- causal,
999
- startPos,
1000
- attnSoftcap,
1001
- slidingWindow,
1002
- });
1003
-
1004
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 0, 'uniforms', uniformBuffer);
1005
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 1, 'Q', Q?.buffer, [
1006
- `QLabel=${Q?.label ?? 'unknown'}`,
1007
- `QDtype=${Q?.dtype ?? 'unknown'}`,
1008
- ]);
1009
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 2, 'basisK', basisK?.buffer, [
1010
- `basisKLabel=${basisK?.label ?? 'unknown'}`,
1011
- `basisKDtype=${basisK?.dtype ?? 'unknown'}`,
1012
- ]);
1013
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 3, 'basisV', basisV?.buffer, [
1014
- `basisVLabel=${basisV?.label ?? 'unknown'}`,
1015
- `basisVDtype=${basisV?.dtype ?? 'unknown'}`,
1016
- ]);
1017
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 4, 'pagedK', pagedK);
1018
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 5, 'pagedV', pagedV);
1019
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 6, 'index', index);
1020
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 7, 'ropeCos', ropeCos);
1021
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 8, 'ropeSin', ropeSin);
1022
- assertAttentionBindGroupBuffer('attention_bdpa', variant, 9, 'output', outputBuf);
1023
-
1024
- const bindGroup = execution.device.createBindGroup({
1025
- label: 'attention_bdpa_bind_group',
1026
- layout: getPipelineBindGroupLayout(pipeline, 0),
1027
- entries: [
1028
- { binding: 0, resource: { buffer: uniformBuffer } },
1029
- { binding: 1, resource: { buffer: Q.buffer } },
1030
- { binding: 2, resource: { buffer: basisK.buffer } },
1031
- { binding: 3, resource: { buffer: basisV.buffer } },
1032
- { binding: 4, resource: { buffer: pagedK } },
1033
- { binding: 5, resource: { buffer: pagedV } },
1034
- { binding: 6, resource: { buffer: index } },
1035
- { binding: 7, resource: { buffer: ropeCos } },
1036
- { binding: 8, resource: { buffer: ropeSin } },
1037
- { binding: 9, resource: { buffer: outputBuf } },
1038
- ],
1039
- });
1040
-
1041
- dispatchAttentionKernel(execution, kernel, pipeline, bindGroup, numHeads);
1042
- releaseAttentionUniform(execution, uniformBuffer);
1043
-
1044
- return createTensor(outputBuf, outputDtype, [seqLen, numHeads, headDim], 'attention_bdpa_output');
1045
- }
1046
-
1047
- async function executeAttention(
1048
- recorder,
1049
- Q,
1050
- K,
1051
- V,
1052
- mask,
1053
- numHeads,
1054
- headDim,
1055
- options = {}
1056
- ) {
1057
- const execution = resolveAttentionExecution(recorder);
1058
- const {
1059
- seqLen = 1,
1060
- kvLen = seqLen,
1061
- numKVHeads = numHeads,
1062
- scale = 1.0 / Math.sqrt(headDim),
1063
- causal = true,
1064
- bidirectionalSpanStart = 0,
1065
- bidirectionalSpanLength = 0,
1066
- startPos = 0,
1067
- layerIdx,
1068
- outputBuffer = null,
1069
- attnSoftcap = 0,
1070
- slidingWindow = 0,
1071
- kvLenBuffer = null,
1072
- indirectBuffer = null,
1073
- indirectOffset = 0,
1074
- kvStart = 0,
1075
- kvLayout = 'contiguous',
1076
- kvPageTable = null,
1077
- kvPageSize = 0,
1078
- kernelPath = null,
1079
- outputGate = null,
1080
- } = options;
1081
- if (!Number.isFinite(bidirectionalSpanStart) || Math.floor(bidirectionalSpanStart) !== bidirectionalSpanStart || bidirectionalSpanStart < 0) {
1082
- throw new Error(`Attention bidirectionalSpanStart must be a non-negative integer, got ${bidirectionalSpanStart}.`);
1083
- }
1084
- if (!Number.isFinite(bidirectionalSpanLength) || Math.floor(bidirectionalSpanLength) !== bidirectionalSpanLength || bidirectionalSpanLength < 0) {
1085
- throw new Error(`Attention bidirectionalSpanLength must be a non-negative integer, got ${bidirectionalSpanLength}.`);
1086
- }
1087
- if (bidirectionalSpanLength > 0 && (bidirectionalSpanStart + bidirectionalSpanLength) > (kvStart + kvLen)) {
1088
- throw new Error(
1089
- `Attention bidirectional span [${bidirectionalSpanStart}, ${bidirectionalSpanStart + bidirectionalSpanLength}) ` +
1090
- `exceeds KV extent [${kvStart}, ${kvStart + kvLen}).`
1091
- );
1092
- }
1093
-
1094
- // ORT-style single-pass flash attention (adapted from microsoft/onnxruntime
1095
- // flash_attention.wgsl.template). Single kernel, no reduce pass — each WG
1096
- // handles one (head, query-tile) and processes all K online. Gated by
1097
- // useOrtFlashPrefill; takes precedence over the split+reduce flash path.
1098
- if (
1099
- options.useOrtFlashPrefill === true
1100
- && headDim === FLASH_HEAD_DIM
1101
- && seqLen > 1
1102
- && kvLayout === 'contiguous'
1103
- && bidirectionalSpanLength === 0
1104
- && indirectBuffer == null
1105
- && K?.dtype === 'f16'
1106
- && V?.dtype === 'f16'
1107
- ) {
1108
- return executeOrtFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, {
1109
- seqLen,
1110
- kvLen,
1111
- numKVHeads,
1112
- scale,
1113
- causal,
1114
- startPos,
1115
- outputBuffer,
1116
- attnSoftcap,
1117
- slidingWindow,
1118
- kvLenBuffer,
1119
- kvStart,
1120
- kvLayout,
1121
- kvPageTable,
1122
- kvPageSize,
1123
- });
1124
- }
1125
-
1126
- // Flash-attention prefill path: raises RDNA3 occupancy via KV-axis workgroup
1127
- // splitting + online-softmax reduction. Gated by options.useFlashPrefill so
1128
- // callers opt in deliberately (runtime config flag). Conservative conditions:
1129
- // head_dim=256, prefill (seqLen>1), contiguous KV, no bidirectional span.
1130
- if (!globalThis.__DOPPLER_FLASH_TRACE2__ && options.useFlashPrefill === true) {
1131
- globalThis.__DOPPLER_FLASH_TRACE2__ = true;
1132
- if (typeof process !== 'undefined' && process?.stderr?.write) process.stderr.write('[FLASH2] useFlash=' + options.useFlashPrefill
1133
- + ' hd=' + headDim + '/' + FLASH_HEAD_DIM
1134
- + ' seq=' + seqLen + ' kvL=' + kvLayout
1135
- + ' biDir=' + bidirectionalSpanLength
1136
- + ' indirect=' + (indirectBuffer == null ? 'n' : 'y')
1137
- + ' K=' + K?.dtype + ' V=' + V?.dtype + '\n');
1138
- }
1139
- if (
1140
- options.useFlashPrefill === true
1141
- && headDim === FLASH_HEAD_DIM
1142
- && seqLen > 1
1143
- && kvLayout === 'contiguous'
1144
- && bidirectionalSpanLength === 0
1145
- && indirectBuffer == null
1146
- && K?.dtype === 'f16'
1147
- && V?.dtype === 'f16'
1148
- ) {
1149
- return executeFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, {
1150
- seqLen,
1151
- kvLen,
1152
- numKVHeads,
1153
- scale,
1154
- causal,
1155
- startPos,
1156
- outputBuffer,
1157
- attnSoftcap,
1158
- slidingWindow,
1159
- kvLenBuffer,
1160
- kvStart,
1161
- kvLayout,
1162
- kvPageTable,
1163
- kvPageSize,
1164
- });
1165
- }
1166
-
1167
- const limits = getDeviceLimits();
1168
- const sharedLimit = limits?.maxComputeWorkgroupStorageSize ?? Infinity;
1169
- const caps = getKernelCapabilities();
1170
-
1171
- const kvDtype = K.dtype;
1172
- const qDtype = Q.dtype;
1173
- const isPaged = kvLayout === 'paged';
1174
- const plan = resolveAttentionPlan(
1175
- seqLen,
1176
- kvLen,
1177
- headDim,
1178
- numHeads,
1179
- kvDtype,
1180
- qDtype,
1181
- sharedLimit,
1182
- caps,
1183
- layerIdx,
1184
- isPaged,
1185
- kernelPath
1186
- );
1187
-
1188
- if (execution.recorder) {
1189
- trace.attn(0, `recordAttention: isDecode=${plan.isDecode}, tier=${plan.tier}, variant=${plan.variant}, seqLen=${seqLen}, kvLen=${kvLen}, numHeads=${numHeads}, headDim=${headDim}, useF16KV=${plan.useF16KV}`);
1190
- }
1191
-
1192
- const kernel = new AttentionKernel(execution.device);
1193
- const pipelineConstants = resolveOnlineDecodePipelineConstants(plan.variant, headDim, {
1194
- kvLayout,
1195
- slidingWindow,
1196
- });
1197
- const usesOutputGateFusion = plan.variant === 'decode_online_head256_f16kv_output_gate';
1198
- const pipeline = await kernel.getPipeline(plan.variant, pipelineConstants);
1199
-
1200
- const outputConfig = getKernelConfig('attention', plan.variant);
1201
- const outputDtype = outputConfig.outputDtype;
1202
- if (!outputDtype) {
1203
- if (execution.recorder) {
1204
- throw new Error(`Kernel config missing outputDtype for attention variant "${plan.variant}".`);
1205
- }
1206
- throw new Error(`[Attention] outputDtype is required for variant "${plan.variant}".`);
1207
- }
1208
- const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
1209
- const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1210
- const outputSize = seqLen * paddedHiddenSize * bytesPerElement;
1211
- const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_output');
1212
-
1213
- const uniformBuffer = createAttentionUniformBuffer(execution.device, execution.recorder, {
1214
- numHeads,
1215
- numKVHeads,
1216
- headDim,
1217
- kvLen,
1218
- seqLen,
1219
- scale,
1220
- causal,
1221
- startPos,
1222
- attnSoftcap,
1223
- slidingWindow,
1224
- kvLenSource: kvLenBuffer ? 1 : 0,
1225
- kvStart,
1226
- pageSize: kvPageSize,
1227
- kvLayout: kvLayout === 'paged' ? 2 : (kvLayout === 'ring' ? 1 : 0),
1228
- bidirectionalSpanStart,
1229
- bidirectionalSpanLength,
1230
- });
1231
-
1232
- const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(execution.device);
1233
- const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(execution.device);
1234
- const outputGateBinding = usesOutputGateFusion ? outputGate?.buffer : null;
1235
- if (usesOutputGateFusion) {
1236
- if (outputGate?.dtype !== 'f32') {
1237
- throw new Error(`[Attention] outputGate fusion requires f32 gate tensor; got ${String(outputGate?.dtype)}.`);
1238
- }
1239
- const requiredGateBytes = numHeads * headDim * F32_BYTES;
1240
- const actualGateBytes = outputGate?.buffer?.size;
1241
- if (Number.isFinite(actualGateBytes) && actualGateBytes < requiredGateBytes) {
1242
- throw new Error(
1243
- `[Attention] outputGate fusion requires at least ${requiredGateBytes} gate bytes; got ${actualGateBytes}.`
1244
- );
1245
- }
1246
- }
1247
- assertAttentionBindGroupBuffer('attention', plan.variant, 0, 'uniforms', uniformBuffer);
1248
- assertAttentionBindGroupBuffer('attention', plan.variant, 1, 'Q', Q?.buffer, [
1249
- `QLabel=${Q?.label ?? 'unknown'}`,
1250
- `QDtype=${Q?.dtype ?? 'unknown'}`,
1251
- ]);
1252
- assertAttentionBindGroupBuffer('attention', plan.variant, 2, 'K', K?.buffer, [
1253
- `KLabel=${K?.label ?? 'unknown'}`,
1254
- `KDtype=${K?.dtype ?? 'unknown'}`,
1255
- ]);
1256
- assertAttentionBindGroupBuffer('attention', plan.variant, 3, 'V', V?.buffer, [
1257
- `VLabel=${V?.label ?? 'unknown'}`,
1258
- `VDtype=${V?.dtype ?? 'unknown'}`,
1259
- ]);
1260
- assertAttentionBindGroupBuffer('attention', plan.variant, 4, 'output', outputBuf);
1261
- assertAttentionBindGroupBuffer('attention', plan.variant, 5, 'kvLen', kvLenBinding);
1262
- assertAttentionBindGroupBuffer('attention', plan.variant, 6, 'pageTable', pageTableBinding, [
1263
- `kvLayout=${kvLayout}`,
1264
- ]);
1265
- if (usesOutputGateFusion) {
1266
- assertAttentionBindGroupBuffer('attention', plan.variant, 7, 'outputGate', outputGateBinding, [
1267
- `useOutputGateFusion=${usesOutputGateFusion}`,
1268
- ]);
1269
- }
1270
- const bindGroupEntries = [
1271
- { binding: 0, resource: { buffer: uniformBuffer } },
1272
- { binding: 1, resource: { buffer: Q.buffer } },
1273
- { binding: 2, resource: { buffer: K.buffer } },
1274
- { binding: 3, resource: { buffer: V.buffer } },
1275
- { binding: 4, resource: { buffer: outputBuf } },
1276
- { binding: 5, resource: { buffer: kvLenBinding } },
1277
- { binding: 6, resource: { buffer: pageTableBinding } },
1278
- ];
1279
- if (usesOutputGateFusion) {
1280
- bindGroupEntries.push({ binding: 7, resource: { buffer: outputGateBinding } });
1281
- }
1282
- const bindGroup = execution.device.createBindGroup({
1283
- label: 'attention_bind_group',
1284
- layout: getPipelineBindGroupLayout(pipeline, 0),
1285
- entries: bindGroupEntries,
1286
- });
1287
-
1288
- if (!indirectBuffer && limits && plan.workgroups > limits.maxComputeWorkgroupsPerDimension) {
1289
- throw new Error(
1290
- `Attention dispatch requires ${plan.workgroups} workgroups but device limit is ` +
1291
- `${limits.maxComputeWorkgroupsPerDimension}. Reduce prompt length or use streaming attention.`
1292
- );
79
+ workgroups
80
+ ) {
81
+ this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_tiered_quant');
1293
82
  }
1294
83
 
1295
- if (indirectBuffer) {
1296
- if (execution.recorder) {
1297
- recordDispatchIndirect(execution.recorder, pipeline, bindGroup, indirectBuffer, indirectOffset, 'attention');
1298
- } else {
1299
- dispatchIndirect(execution.device, pipeline, bindGroup, indirectBuffer, indirectOffset, 'attention');
1300
- }
1301
- } else {
1302
- dispatchAttentionKernel(execution, kernel, pipeline, bindGroup, plan.workgroups);
84
+ record(
85
+ recorder,
86
+ pipeline,
87
+ bindGroup,
88
+ workgroups
89
+ ) {
90
+ this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_tiered_quant');
1303
91
  }
1304
-
1305
- releaseAttentionUniform(execution, uniformBuffer);
1306
-
1307
- const outputTensor = createTensor(outputBuf, outputDtype, [seqLen, numHeads, headDim], 'attention_output');
1308
- outputTensor.outputGateFused = usesOutputGateFusion;
1309
- return outputTensor;
1310
92
  }
1311
93
 
1312
- // -----------------------------------------------------------------------------
1313
- // Flash-attention prefill path (head_dim = 256, f16 KV)
1314
- // -----------------------------------------------------------------------------
1315
- // Two-pass kernel to raise RDNA3 occupancy: pass 1 processes one KV slice per
1316
- // workgroup and writes per-split (acc, m, l) partials; pass 2 merges across
1317
- // splits with online softmax. Single recorder, two dispatches — queue order
1318
- // handles the read-after-write between passes.
1319
-
1320
- const FLASH_BLOCK_SIZE = 32;
1321
- const FLASH_HEAD_DIM = 256;
1322
- const FLASH_HEAD_DIM_VECS = 64;
1323
- const FLASH_REDUCE_WG = 64;
1324
-
1325
- let flashPrefillKernel = null;
1326
- let flashReduceKernel = null;
94
+ class AttentionContiguousQuantKernel extends KernelBase {
1327
95
 
1328
- class FlashAttentionPrefillKernel extends KernelBase {
1329
96
  async getPipeline(variant) {
1330
- return this.getPipelineFor('attention', variant);
1331
- }
1332
- record(recorder, pipeline, bindGroup, workgroups) {
1333
- this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention');
97
+ return this.getPipelineFor('attention_contiguous_quant', variant);
1334
98
  }
1335
- dispatch(pipeline, bindGroup, workgroups) {
1336
- this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention');
99
+
100
+ dispatch(
101
+ pipeline,
102
+ bindGroup,
103
+ workgroups
104
+ ) {
105
+ this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
1337
106
  }
1338
- }
1339
107
 
1340
- function getFlashPrefillKernel(device) {
1341
- if (!flashPrefillKernel) {
1342
- flashPrefillKernel = new FlashAttentionPrefillKernel(device);
108
+ record(
109
+ recorder,
110
+ pipeline,
111
+ bindGroup,
112
+ workgroups
113
+ ) {
114
+ this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
1343
115
  }
1344
- return flashPrefillKernel;
1345
116
  }
1346
117
 
1347
- function getFlashReduceKernel(device) {
1348
- if (!flashReduceKernel) {
1349
- flashReduceKernel = new FlashAttentionPrefillKernel(device);
1350
- }
1351
- return flashReduceKernel;
118
+ // Track if we've logged chunked kernel selection
119
+
120
+ export function resolveAttentionPlanForTest(
121
+ seqLen,
122
+ kvLen,
123
+ headDim,
124
+ numHeads,
125
+ kvDtype,
126
+ qDtype,
127
+ sharedLimit,
128
+ caps,
129
+ layerIdx,
130
+ isPaged = false,
131
+ kernelPath = null
132
+ ) {
133
+ return resolveAttentionPlan(
134
+ seqLen,
135
+ kvLen,
136
+ headDim,
137
+ numHeads,
138
+ kvDtype,
139
+ qDtype,
140
+ sharedLimit,
141
+ caps,
142
+ layerIdx,
143
+ isPaged,
144
+ kernelPath
145
+ );
1352
146
  }
1353
147
 
1354
- function createFlashAttentionUniformBuffer(device, recorder, params) {
1355
- // Layout mirrors the flash kernel's Uniforms struct (see
1356
- // attention_prefill_flash_head256_f16kv.wgsl). 64 bytes total.
148
+ function createTieredAttentionUniformBuffer(
149
+ device,
150
+ recorder,
151
+ params
152
+ ) {
1357
153
  return createUniformBufferWithView(
1358
- 'attention_flash_uniforms',
1359
- 64,
154
+ 'attention_tiered_uniforms',
155
+ 80,
1360
156
  (view) => {
1361
157
  view.setUint32(0, params.numHeads, true);
1362
158
  view.setUint32(4, params.numKVHeads, true);
1363
159
  view.setUint32(8, params.headDim, true);
1364
- view.setUint32(12, params.kvLen, true);
1365
- view.setUint32(16, params.seqLen, true);
1366
- view.setFloat32(20, params.scale, true);
1367
- view.setUint32(24, params.causal ? 1 : 0, true);
1368
- view.setUint32(28, params.startPos, true);
1369
- view.setFloat32(32, params.attnSoftcap, true);
1370
- view.setUint32(36, params.slidingWindow, true);
1371
- view.setUint32(40, params.kvLenSource, true);
1372
- view.setUint32(44, params.kvStart ?? 0, true);
1373
- view.setUint32(48, params.pageSize ?? 0, true);
1374
- view.setUint32(52, params.kvLayout ?? 0, true);
1375
- view.setUint32(56, params.numKvSplits, true);
1376
- view.setUint32(60, 0, true);
160
+ view.setUint32(12, params.coldLen, true);
161
+ view.setUint32(16, params.hotLen, true);
162
+ view.setUint32(20, params.seqLen, true);
163
+ view.setFloat32(24, params.scale, true);
164
+ view.setUint32(28, params.causal ? 1 : 0, true);
165
+ view.setUint32(32, params.startPos, true);
166
+ view.setFloat32(36, params.attnSoftcap, true);
167
+ view.setUint32(40, params.slidingWindow, true);
168
+ view.setUint32(44, params.hotWindow, true);
169
+ view.setUint32(48, params.hotStart, true);
170
+ view.setUint32(52, params.coldPageSize, true);
171
+ view.setUint32(56, params.coldLayout ?? 0, true);
172
+ view.setUint32(60, params.hotLayout ?? 1, true);
173
+ view.setUint32(64, 0, true);
1377
174
  },
1378
175
  recorder,
1379
176
  device
1380
177
  );
1381
178
  }
1382
179
 
1383
- function createFlashReduceUniformBuffer(device, recorder, params) {
180
+ function createTieredQuantAttentionUniformBuffer(
181
+ device,
182
+ recorder,
183
+ params
184
+ ) {
1384
185
  return createUniformBufferWithView(
1385
- 'attention_flash_reduce_uniforms',
1386
- 16,
186
+ 'attention_tiered_quant_uniforms',
187
+ 64,
1387
188
  (view) => {
1388
189
  view.setUint32(0, params.numHeads, true);
1389
- view.setUint32(4, params.queryLen, true);
1390
- view.setUint32(8, params.numKvSplits, true);
1391
- view.setUint32(12, 0, true);
190
+ view.setUint32(4, params.numKVHeads, true);
191
+ view.setUint32(8, params.headDim, true);
192
+ view.setUint32(12, params.coldLen, true);
193
+ view.setUint32(16, params.hotLen, true);
194
+ view.setUint32(20, params.seqLen, true);
195
+ view.setFloat32(24, params.scale, true);
196
+ view.setUint32(28, params.causal ? 1 : 0, true);
197
+ view.setUint32(32, params.startPos, true);
198
+ view.setFloat32(36, params.attnSoftcap, true);
199
+ view.setUint32(40, params.slidingWindow, true);
200
+ view.setUint32(44, params.hotWindow, true);
201
+ view.setUint32(48, params.hotStart, true);
202
+ view.setUint32(52, params.packedStride, true);
203
+ view.setUint32(56, 0, true);
1392
204
  },
1393
205
  recorder,
1394
206
  device
1395
207
  );
1396
208
  }
1397
209
 
1398
- function chooseFlashNumKvSplits(kvLen) {
1399
- // Target roughly 32 workgroups × num_heads × num_kv_splits ≈ 4x RDNA3
1400
- // compute-unit count. Keep at least 2× FLASH_BLOCK_SIZE KV positions per
1401
- // split so each workgroup has enough work to amortise dispatch overhead.
1402
- // Short prefills (kvLen ≤ 2 × BLOCK_SIZE) take the single-split fast path
1403
- // which skips the reduce pass entirely.
1404
- if (kvLen <= 2 * FLASH_BLOCK_SIZE) return 1;
1405
- const maxSplits = Math.min(8, Math.floor(kvLen / (2 * FLASH_BLOCK_SIZE)));
1406
- return Math.max(1, maxSplits);
1407
- }
1408
-
1409
- async function executeFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, options = {}) {
1410
- if (headDim !== FLASH_HEAD_DIM) {
1411
- throw new Error(`[FlashAttention] headDim must be ${FLASH_HEAD_DIM}, got ${headDim}.`);
1412
- }
1413
- const execution = resolveAttentionExecution(recorder);
1414
- const {
1415
- seqLen = 1,
1416
- kvLen = seqLen,
1417
- numKVHeads = numHeads,
1418
- scale = 1.0 / Math.sqrt(headDim),
1419
- causal = true,
1420
- startPos = 0,
1421
- outputBuffer = null,
1422
- attnSoftcap = 0,
1423
- slidingWindow = 0,
1424
- kvLenBuffer = null,
1425
- kvStart = 0,
1426
- kvLayout = 'contiguous',
1427
- kvPageTable = null,
1428
- kvPageSize = 0,
1429
- } = options;
1430
-
1431
- if (kvLayout !== 'contiguous') {
1432
- throw new Error(`[FlashAttention] kvLayout must be "contiguous", got "${kvLayout}".`);
1433
- }
1434
-
1435
- const device = execution.device;
1436
- const numQueryBlocks = Math.max(1, Math.ceil(seqLen / FLASH_BLOCK_SIZE));
1437
- const numKvSplits = chooseFlashNumKvSplits(kvLen);
1438
- const singleSplit = numKvSplits === 1;
1439
-
1440
- // Final output buffer (always allocated; bound to slot 4 on the single-split
1441
- // fast path where the kernel writes normalised output directly).
1442
- const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1443
- const outputSize = seqLen * paddedHiddenSize * 4;
1444
- const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_flash_output');
1445
-
1446
- // Intermediate buffers for the multi-split path. On the single-split fast
1447
- // path we bypass the reduce pass entirely and bind the output buffer in
1448
- // slot 4 (partial_acc slot) while m/l bindings get tiny stub buffers — the
1449
- // kernel skips writes to them.
1450
- const partialAccBytes = singleSplit
1451
- ? 4
1452
- : numQueryBlocks * numHeads * numKvSplits * FLASH_BLOCK_SIZE * FLASH_HEAD_DIM * 4;
1453
- const partialStatsBytes = singleSplit
1454
- ? 4
1455
- : numQueryBlocks * numHeads * numKvSplits * FLASH_BLOCK_SIZE * 4;
1456
- const partialAcc = singleSplit
1457
- ? outputBuf
1458
- : acquireBuffer(partialAccBytes, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST, 'flash_partial_acc');
1459
- const partialM = acquireBuffer(
1460
- partialStatsBytes,
1461
- GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1462
- 'flash_partial_m'
1463
- );
1464
- const partialL = acquireBuffer(
1465
- partialStatsBytes,
1466
- GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1467
- 'flash_partial_l'
1468
- );
1469
-
1470
- // Pass 1 uniforms + dispatch.
1471
- const flashUniform = createFlashAttentionUniformBuffer(device, execution.recorder, {
1472
- numHeads,
1473
- numKVHeads,
1474
- headDim,
1475
- kvLen,
1476
- seqLen,
1477
- scale,
1478
- causal,
1479
- startPos,
1480
- attnSoftcap,
1481
- slidingWindow,
1482
- kvLenSource: kvLenBuffer ? 1 : 0,
1483
- kvStart,
1484
- pageSize: kvPageSize,
1485
- kvLayout: 0, // contiguous only for now
1486
- numKvSplits,
1487
- });
1488
-
1489
- const flashKernel = getFlashPrefillKernel(device);
1490
- const flashPipeline = await flashKernel.getPipeline('prefill_flash_head256_f16kv');
1491
- const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(device);
1492
- const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(device);
1493
-
1494
- const flashBindGroup = device.createBindGroup({
1495
- label: 'attention_flash_prefill_bg',
1496
- layout: flashPipeline.getBindGroupLayout(0),
1497
- entries: [
1498
- { binding: 0, resource: { buffer: flashUniform } },
1499
- { binding: 1, resource: { buffer: Q.buffer } },
1500
- { binding: 2, resource: { buffer: K.buffer } },
1501
- { binding: 3, resource: { buffer: V.buffer } },
1502
- { binding: 4, resource: { buffer: partialAcc } },
1503
- { binding: 5, resource: { buffer: partialM } },
1504
- { binding: 6, resource: { buffer: partialL } },
1505
- { binding: 7, resource: { buffer: kvLenBinding } },
1506
- { binding: 8, resource: { buffer: pageTableBinding } },
1507
- ],
1508
- });
1509
-
1510
- dispatchAttentionKernel(
1511
- execution,
1512
- flashKernel,
1513
- flashPipeline,
1514
- flashBindGroup,
1515
- numQueryBlocks * numHeads * numKvSplits
1516
- );
1517
- releaseAttentionUniform(execution, flashUniform);
1518
-
1519
- // Pass 2 — reduce. Skipped on the single-split fast path where pass 1
1520
- // already wrote the final normalised output directly to outputBuf.
1521
- if (!singleSplit) {
1522
- const reduceUniform = createFlashReduceUniformBuffer(device, execution.recorder, {
1523
- numHeads,
1524
- queryLen: seqLen,
1525
- numKvSplits,
1526
- });
1527
-
1528
- const reduceKernel = getFlashReduceKernel(device);
1529
- const reducePipeline = await reduceKernel.getPipeline('prefill_flash_reduce');
1530
-
1531
- const reduceBindGroup = device.createBindGroup({
1532
- label: 'attention_flash_reduce_bg',
1533
- layout: reducePipeline.getBindGroupLayout(0),
1534
- entries: [
1535
- { binding: 0, resource: { buffer: reduceUniform } },
1536
- { binding: 1, resource: { buffer: partialAcc } },
1537
- { binding: 2, resource: { buffer: partialM } },
1538
- { binding: 3, resource: { buffer: partialL } },
1539
- { binding: 4, resource: { buffer: outputBuf } },
1540
- ],
1541
- });
1542
-
1543
- const totalQh = seqLen * numHeads;
1544
- const reduceWgX = Math.ceil(totalQh / FLASH_REDUCE_WG);
1545
- dispatchAttentionKernel(
1546
- execution,
1547
- reduceKernel,
1548
- reducePipeline,
1549
- reduceBindGroup,
1550
- [reduceWgX, FLASH_HEAD_DIM_VECS, 1]
1551
- );
1552
- releaseAttentionUniform(execution, reduceUniform);
1553
- }
1554
-
1555
- // Release intermediate buffers via the recorder's deferred cleanup so GPU
1556
- // work completes before they re-enter the pool. On the single-split path
1557
- // partialAcc IS the output buffer, so we skip it.
1558
- const intermediates = singleSplit ? [partialM, partialL] : [partialAcc, partialM, partialL];
1559
- if (execution.recorder) {
1560
- for (const buf of intermediates) {
1561
- execution.recorder.trackTemporaryBuffer(buf);
1562
- }
1563
- } else {
1564
- device.queue.onSubmittedWorkDone().then(() => {
1565
- for (const buf of intermediates) {
1566
- releaseBuffer(buf);
210
+ function createContiguousQuantAttentionUniformBuffer(
211
+ device,
212
+ recorder,
213
+ params
214
+ ) {
215
+ const hasProdFields = params.packedStrideMSE != null;
216
+ const size = hasProdFields ? 64 : 48;
217
+ return createUniformBufferWithView(
218
+ 'attention_contiguous_quant_uniforms',
219
+ size,
220
+ (view) => {
221
+ view.setUint32(0, params.numHeads, true);
222
+ view.setUint32(4, params.numKVHeads, true);
223
+ view.setUint32(8, params.headDim, true);
224
+ view.setUint32(12, params.kvLen, true);
225
+ view.setUint32(16, params.seqLen, true);
226
+ view.setFloat32(20, params.scale, true);
227
+ view.setUint32(24, params.causal ? 1 : 0, true);
228
+ view.setUint32(28, params.startPos, true);
229
+ view.setFloat32(32, params.attnSoftcap, true);
230
+ view.setUint32(36, params.slidingWindow, true);
231
+ if (hasProdFields) {
232
+ view.setUint32(40, params.packedStrideMSE, true);
233
+ view.setUint32(44, params.packedStrideResidual, true);
234
+ view.setUint32(48, 0, true);
235
+ view.setUint32(52, 0, true);
236
+ view.setUint32(56, 0, true);
237
+ view.setUint32(60, 0, true);
238
+ } else {
239
+ view.setUint32(40, params.packedStride, true);
240
+ view.setUint32(44, 0, true);
1567
241
  }
1568
- });
1569
- }
1570
-
1571
- return createTensor(outputBuf, 'f32', [seqLen, numHeads, headDim], 'attention_flash_output');
242
+ },
243
+ recorder,
244
+ device
245
+ );
1572
246
  }
1573
247
 
248
+ // -----------------------------------------------------------------------------
249
+ // Flash-attention prefill path (head_dim = 256, f16 KV)
250
+ // -----------------------------------------------------------------------------
251
+ // Two-pass kernel to raise RDNA3 occupancy: pass 1 processes one KV slice per
252
+ // workgroup and writes per-split (acc, m, l) partials; pass 2 merges across
253
+ // splits with online softmax. Single recorder, two dispatches — queue order
254
+ // handles the read-after-write between passes.
255
+
1574
256
  // Single-pass flash-attention dispatcher (ORT-style). 7-binding contract —
1575
257
  // same as attention_head256_f16kv. One kernel launch, no reduce pass.
1576
258
  // Workgroups: (num_heads, ceil(seqLen / ORT_FLASH_WG), 1) with ORT_FLASH_WG=64.
1577
- const ORT_FLASH_WG = 64;
1578
-
1579
- async function executeOrtFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, options = {}) {
1580
- if (headDim !== FLASH_HEAD_DIM) {
1581
- throw new Error(`[OrtFlashAttention] headDim must be ${FLASH_HEAD_DIM}, got ${headDim}.`);
1582
- }
1583
- const execution = resolveAttentionExecution(recorder);
1584
- const {
1585
- seqLen = 1,
1586
- kvLen = seqLen,
1587
- numKVHeads = numHeads,
1588
- scale = 1.0 / Math.sqrt(headDim),
1589
- causal = true,
1590
- startPos = 0,
1591
- outputBuffer = null,
1592
- attnSoftcap = 0,
1593
- slidingWindow = 0,
1594
- kvLenBuffer = null,
1595
- kvStart = 0,
1596
- kvLayout = 'contiguous',
1597
- kvPageTable = null,
1598
- kvPageSize = 0,
1599
- } = options;
1600
-
1601
- if (kvLayout !== 'contiguous') {
1602
- throw new Error(`[OrtFlashAttention] kvLayout must be "contiguous", got "${kvLayout}".`);
1603
- }
1604
-
1605
- const device = execution.device;
1606
- const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1607
- const outputSize = seqLen * paddedHiddenSize * 4;
1608
- const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_ort_flash_output');
1609
-
1610
- const uniform = createAttentionUniformBuffer(device, execution.recorder, {
1611
- numHeads,
1612
- numKVHeads,
1613
- headDim,
1614
- kvLen,
1615
- seqLen,
1616
- scale,
1617
- causal,
1618
- startPos,
1619
- attnSoftcap,
1620
- slidingWindow,
1621
- kvLenSource: kvLenBuffer ? 1 : 0,
1622
- kvStart,
1623
- pageSize: kvPageSize,
1624
- kvLayout: 0, // contiguous only
1625
- });
1626
-
1627
- const kernel = new AttentionKernel(device);
1628
- const pipeline = await kernel.getPipeline('prefill_flash_ort_head256_f16kv');
1629
- const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(device);
1630
- const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(device);
1631
-
1632
- const bindGroup = execution.device.createBindGroup({
1633
- label: 'attention_ort_flash_prefill_bg',
1634
- layout: getPipelineBindGroupLayout(pipeline, 0),
1635
- entries: [
1636
- { binding: 0, resource: { buffer: uniform } },
1637
- { binding: 1, resource: { buffer: Q.buffer } },
1638
- { binding: 2, resource: { buffer: K.buffer } },
1639
- { binding: 3, resource: { buffer: V.buffer } },
1640
- { binding: 4, resource: { buffer: outputBuf } },
1641
- { binding: 5, resource: { buffer: kvLenBinding } },
1642
- { binding: 6, resource: { buffer: pageTableBinding } },
1643
- ],
1644
- });
1645
-
1646
- const numSeqTiles = Math.max(1, Math.ceil(seqLen / ORT_FLASH_WG));
1647
- const workgroups = [numHeads, numSeqTiles, 1];
1648
-
1649
- if (execution.recorder) {
1650
- kernel.record(execution.recorder, pipeline, bindGroup, workgroups);
1651
- } else {
1652
- kernel.dispatch(pipeline, bindGroup, workgroups);
1653
- }
1654
-
1655
- if (uniform) releaseUniformBuffer(uniform);
1656
- return createTensor(outputBuf, 'f32', [seqLen, numHeads, headDim], 'attention_ort_flash_output');
1657
- }
1658
259
 
1659
260
  async function executeAttentionTiered(
1660
261
  recorder,
@@ -2111,7 +712,6 @@ export async function recordAttentionTieredQuant(
2111
712
  );
2112
713
  }
2113
714
 
2114
-
2115
715
  // =============================================================================
2116
716
  // Contiguous Quantized Attention (TurboQuant for full-attention models)
2117
717
  // =============================================================================