doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,3 +1,5 @@
1
+ override WORKGROUP_SIZE: u32 = 256u;
2
+
1
3
  struct Uniforms {
2
4
  size: u32,
3
5
  _pad0: u32,
@@ -11,7 +13,7 @@ struct Uniforms {
11
13
  @group(0) @binding(1) var<storage, read> input: array<f32>;
12
14
  @group(0) @binding(2) var<storage, read_write> output: array<f32>;
13
15
 
14
- @compute @workgroup_size(256, 1, 1)
16
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
15
17
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
16
18
  let idx = gid.x;
17
19
  if (idx >= u.size) {
@@ -96,25 +96,6 @@ export interface BDPAAttentionOptions extends OutputBufferOptions {
96
96
 
97
97
  export type AttentionTier = 'subgroup' | 'tiled_large' | 'tiled_small' | 'streaming';
98
98
 
99
- /** Context for attention tier selection rules. */
100
- export interface AttentionTierContext {
101
- canSubgroup: boolean;
102
- canLarge: boolean;
103
- canSmall: boolean;
104
- isDecode: boolean;
105
- }
106
-
107
- /** Context for attention variant selection rules. */
108
- export interface AttentionVariantContext {
109
- tier: AttentionTier;
110
- useF16KV: boolean;
111
- canUseChunked: boolean;
112
- canUseDecodeSubgroup: boolean;
113
- canUseDecodeOptimized?: boolean;
114
- isPaged?: boolean;
115
- isDecode?: boolean;
116
- }
117
-
118
99
  /**
119
100
  * Run attention operation
120
101
  */
@@ -236,3 +217,31 @@ export declare function resolveAttentionPlanForTest(
236
217
  useF16KV: boolean;
237
218
  isDecode: boolean;
238
219
  };
220
+
221
+ /**
222
+ * Run contiguous-quantized attention (submit + wait).
223
+ * Types mirror runAttentionTieredQuant above.
224
+ */
225
+ export declare function runAttentionContiguousQuant(
226
+ Q: Tensor,
227
+ packedK: GPUBuffer,
228
+ packedV: GPUBuffer,
229
+ scalesK: GPUBuffer,
230
+ scalesV: GPUBuffer,
231
+ numHeads: number,
232
+ headDim: number,
233
+ options?: AttentionOptions
234
+ ): Promise<Tensor>;
235
+
236
+ /** Record contiguous-quantized attention onto a CommandRecorder. */
237
+ export declare function recordAttentionContiguousQuant(
238
+ recorder: CommandRecorder,
239
+ Q: Tensor,
240
+ packedK: GPUBuffer,
241
+ packedV: GPUBuffer,
242
+ scalesK: GPUBuffer,
243
+ scalesV: GPUBuffer,
244
+ numHeads: number,
245
+ headDim: number,
246
+ options?: AttentionOptions
247
+ ): Promise<Tensor>;
@@ -1,7 +1,7 @@
1
1
 
2
2
 
3
3
  import { getDevice, getDeviceEpoch, getDeviceLimits, getKernelCapabilities } from '../device.js';
4
- import { acquireBuffer } from '../../memory/buffer-pool.js';
4
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
5
5
  import { createTensor } from '../tensor.js';
6
6
  import { KernelBase } from './kernel-base.js';
7
7
  import { TILE_SIZES } from './constants.js';
@@ -351,17 +351,31 @@ function resolveAttentionVariant(
351
351
  }
352
352
 
353
353
 
354
- function calculateAttentionWorkgroups(tier, seqLen, numHeads) {
354
+ function resolveAttentionQueryBlockSize(tier, variant = null) {
355
+ if (variant) {
356
+ const metadataBlockSize = getKernelConfig('attention', variant).variantMetadata?.queryBlockSize;
357
+ if (metadataBlockSize != null) {
358
+ if (!Number.isInteger(metadataBlockSize) || metadataBlockSize <= 0) {
359
+ throw new Error(`Attention kernel "${variant}" has invalid variantMetadata.queryBlockSize=${metadataBlockSize}.`);
360
+ }
361
+ return metadataBlockSize;
362
+ }
363
+ }
364
+ if (tier === 'tiled_large') {
365
+ return TILE_SIZES.ATTENTION_LARGE_BLOCK_SIZE;
366
+ }
367
+ return TILE_SIZES.ATTENTION_SMALL_BLOCK_SIZE;
368
+ }
369
+
370
+ function calculateAttentionWorkgroups(tier, seqLen, numHeads, variant = null) {
355
371
  if (tier === 'subgroup') {
356
372
  return numHeads;
357
373
  }
358
374
  if (tier === 'streaming') {
359
375
  return seqLen * numHeads;
360
376
  }
361
- if (tier === 'tiled_large') {
362
- return Math.ceil(seqLen / TILE_SIZES.ATTENTION_LARGE_BLOCK_SIZE) * numHeads;
363
- }
364
- return Math.ceil(seqLen / TILE_SIZES.ATTENTION_SMALL_BLOCK_SIZE) * numHeads;
377
+ const queryBlockSize = resolveAttentionQueryBlockSize(tier, variant);
378
+ return Math.ceil(seqLen / queryBlockSize) * numHeads;
365
379
  }
366
380
 
367
381
 
@@ -561,7 +575,7 @@ function resolveAttentionPlan(
561
575
  caps,
562
576
  sharedLimit
563
577
  );
564
- const workgroups = calculateAttentionWorkgroups(adaptiveSelection.tier, seqLen, numHeads);
578
+ const workgroups = calculateAttentionWorkgroups(adaptiveSelection.tier, seqLen, numHeads, adaptiveVariant);
565
579
  logKernelSelectionOnce('attention', {
566
580
  variant: adaptiveVariant,
567
581
  reason: `path_override_fallback:${adaptiveSelection.tier}`,
@@ -603,7 +617,7 @@ function resolveAttentionPlan(
603
617
  }
604
618
 
605
619
  const tier = inferAttentionTierFromVariant(variantOverride);
606
- const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads);
620
+ const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads, variantOverride);
607
621
  logKernelSelectionOnce('attention', {
608
622
  variant: variantOverride,
609
623
  reason: `${selectionReason}:${tier}`,
@@ -635,7 +649,7 @@ function resolveAttentionPlan(
635
649
  kvLen,
636
650
  sharedLimit
637
651
  );
638
- const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads);
652
+ const workgroups = calculateAttentionWorkgroups(tier, seqLen, numHeads, variant);
639
653
 
640
654
  logKernelSelectionOnce('attention', {
641
655
  variant: validatedVariant,
@@ -1039,6 +1053,79 @@ async function executeAttention(
1039
1053
  );
1040
1054
  }
1041
1055
 
1056
+ // ORT-style single-pass flash attention (adapted from microsoft/onnxruntime
1057
+ // flash_attention.wgsl.template). Single kernel, no reduce pass — each WG
1058
+ // handles one (head, query-tile) and processes all K online. Gated by
1059
+ // useOrtFlashPrefill; takes precedence over the split+reduce flash path.
1060
+ if (
1061
+ options.useOrtFlashPrefill === true
1062
+ && headDim === FLASH_HEAD_DIM
1063
+ && seqLen > 1
1064
+ && kvLayout === 'contiguous'
1065
+ && bidirectionalSpanLength === 0
1066
+ && indirectBuffer == null
1067
+ && K?.dtype === 'f16'
1068
+ && V?.dtype === 'f16'
1069
+ ) {
1070
+ return executeOrtFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, {
1071
+ seqLen,
1072
+ kvLen,
1073
+ numKVHeads,
1074
+ scale,
1075
+ causal,
1076
+ startPos,
1077
+ outputBuffer,
1078
+ attnSoftcap,
1079
+ slidingWindow,
1080
+ kvLenBuffer,
1081
+ kvStart,
1082
+ kvLayout,
1083
+ kvPageTable,
1084
+ kvPageSize,
1085
+ });
1086
+ }
1087
+
1088
+ // Flash-attention prefill path: raises RDNA3 occupancy via KV-axis workgroup
1089
+ // splitting + online-softmax reduction. Gated by options.useFlashPrefill so
1090
+ // callers opt in deliberately (runtime config flag). Conservative conditions:
1091
+ // head_dim=256, prefill (seqLen>1), contiguous KV, no bidirectional span.
1092
+ if (!globalThis.__DOPPLER_FLASH_TRACE2__ && options.useFlashPrefill === true) {
1093
+ globalThis.__DOPPLER_FLASH_TRACE2__ = true;
1094
+ if (typeof process !== 'undefined' && process?.stderr?.write) process.stderr.write('[FLASH2] useFlash=' + options.useFlashPrefill
1095
+ + ' hd=' + headDim + '/' + FLASH_HEAD_DIM
1096
+ + ' seq=' + seqLen + ' kvL=' + kvLayout
1097
+ + ' biDir=' + bidirectionalSpanLength
1098
+ + ' indirect=' + (indirectBuffer == null ? 'n' : 'y')
1099
+ + ' K=' + K?.dtype + ' V=' + V?.dtype + '\n');
1100
+ }
1101
+ if (
1102
+ options.useFlashPrefill === true
1103
+ && headDim === FLASH_HEAD_DIM
1104
+ && seqLen > 1
1105
+ && kvLayout === 'contiguous'
1106
+ && bidirectionalSpanLength === 0
1107
+ && indirectBuffer == null
1108
+ && K?.dtype === 'f16'
1109
+ && V?.dtype === 'f16'
1110
+ ) {
1111
+ return executeFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, {
1112
+ seqLen,
1113
+ kvLen,
1114
+ numKVHeads,
1115
+ scale,
1116
+ causal,
1117
+ startPos,
1118
+ outputBuffer,
1119
+ attnSoftcap,
1120
+ slidingWindow,
1121
+ kvLenBuffer,
1122
+ kvStart,
1123
+ kvLayout,
1124
+ kvPageTable,
1125
+ kvPageSize,
1126
+ });
1127
+ }
1128
+
1042
1129
  const limits = getDeviceLimits();
1043
1130
  const sharedLimit = limits?.maxComputeWorkgroupStorageSize ?? Infinity;
1044
1131
  const caps = getKernelCapabilities();
@@ -1155,6 +1242,353 @@ async function executeAttention(
1155
1242
  return createTensor(outputBuf, outputDtype, [seqLen, numHeads, headDim], 'attention_output');
1156
1243
  }
1157
1244
 
1245
+ // -----------------------------------------------------------------------------
1246
+ // Flash-attention prefill path (head_dim = 256, f16 KV)
1247
+ // -----------------------------------------------------------------------------
1248
+ // Two-pass kernel to raise RDNA3 occupancy: pass 1 processes one KV slice per
1249
+ // workgroup and writes per-split (acc, m, l) partials; pass 2 merges across
1250
+ // splits with online softmax. Single recorder, two dispatches — queue order
1251
+ // handles the read-after-write between passes.
1252
+
1253
+ const FLASH_BLOCK_SIZE = 32;
1254
+ const FLASH_HEAD_DIM = 256;
1255
+ const FLASH_HEAD_DIM_VECS = 64;
1256
+ const FLASH_REDUCE_WG = 64;
1257
+
1258
+ let flashPrefillKernel = null;
1259
+ let flashReduceKernel = null;
1260
+
1261
+ class FlashAttentionPrefillKernel extends KernelBase {
1262
+ async getPipeline(variant) {
1263
+ return this.getPipelineFor('attention', variant);
1264
+ }
1265
+ record(recorder, pipeline, bindGroup, workgroups) {
1266
+ this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention');
1267
+ }
1268
+ dispatch(pipeline, bindGroup, workgroups) {
1269
+ this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention');
1270
+ }
1271
+ }
1272
+
1273
+ function getFlashPrefillKernel(device) {
1274
+ if (!flashPrefillKernel) {
1275
+ flashPrefillKernel = new FlashAttentionPrefillKernel(device);
1276
+ }
1277
+ return flashPrefillKernel;
1278
+ }
1279
+
1280
+ function getFlashReduceKernel(device) {
1281
+ if (!flashReduceKernel) {
1282
+ flashReduceKernel = new FlashAttentionPrefillKernel(device);
1283
+ }
1284
+ return flashReduceKernel;
1285
+ }
1286
+
1287
+ function createFlashAttentionUniformBuffer(device, recorder, params) {
1288
+ // Layout mirrors the flash kernel's Uniforms struct (see
1289
+ // attention_prefill_flash_head256_f16kv.wgsl). 64 bytes total.
1290
+ return createUniformBufferWithView(
1291
+ 'attention_flash_uniforms',
1292
+ 64,
1293
+ (view) => {
1294
+ view.setUint32(0, params.numHeads, true);
1295
+ view.setUint32(4, params.numKVHeads, true);
1296
+ view.setUint32(8, params.headDim, true);
1297
+ view.setUint32(12, params.kvLen, true);
1298
+ view.setUint32(16, params.seqLen, true);
1299
+ view.setFloat32(20, params.scale, true);
1300
+ view.setUint32(24, params.causal ? 1 : 0, true);
1301
+ view.setUint32(28, params.startPos, true);
1302
+ view.setFloat32(32, params.attnSoftcap, true);
1303
+ view.setUint32(36, params.slidingWindow, true);
1304
+ view.setUint32(40, params.kvLenSource, true);
1305
+ view.setUint32(44, params.kvStart ?? 0, true);
1306
+ view.setUint32(48, params.pageSize ?? 0, true);
1307
+ view.setUint32(52, params.kvLayout ?? 0, true);
1308
+ view.setUint32(56, params.numKvSplits, true);
1309
+ view.setUint32(60, 0, true);
1310
+ },
1311
+ recorder,
1312
+ device
1313
+ );
1314
+ }
1315
+
1316
+ function createFlashReduceUniformBuffer(device, recorder, params) {
1317
+ return createUniformBufferWithView(
1318
+ 'attention_flash_reduce_uniforms',
1319
+ 16,
1320
+ (view) => {
1321
+ view.setUint32(0, params.numHeads, true);
1322
+ view.setUint32(4, params.queryLen, true);
1323
+ view.setUint32(8, params.numKvSplits, true);
1324
+ view.setUint32(12, 0, true);
1325
+ },
1326
+ recorder,
1327
+ device
1328
+ );
1329
+ }
1330
+
1331
+ function chooseFlashNumKvSplits(kvLen) {
1332
+ // Target roughly 32 workgroups × num_heads × num_kv_splits ≈ 4x RDNA3
1333
+ // compute-unit count. Keep at least 2× FLASH_BLOCK_SIZE KV positions per
1334
+ // split so each workgroup has enough work to amortise dispatch overhead.
1335
+ // Short prefills (kvLen ≤ 2 × BLOCK_SIZE) take the single-split fast path
1336
+ // which skips the reduce pass entirely.
1337
+ if (kvLen <= 2 * FLASH_BLOCK_SIZE) return 1;
1338
+ const maxSplits = Math.min(8, Math.floor(kvLen / (2 * FLASH_BLOCK_SIZE)));
1339
+ return Math.max(1, maxSplits);
1340
+ }
1341
+
1342
+ async function executeFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, options = {}) {
1343
+ if (headDim !== FLASH_HEAD_DIM) {
1344
+ throw new Error(`[FlashAttention] headDim must be ${FLASH_HEAD_DIM}, got ${headDim}.`);
1345
+ }
1346
+ const execution = resolveAttentionExecution(recorder);
1347
+ const {
1348
+ seqLen = 1,
1349
+ kvLen = seqLen,
1350
+ numKVHeads = numHeads,
1351
+ scale = 1.0 / Math.sqrt(headDim),
1352
+ causal = true,
1353
+ startPos = 0,
1354
+ outputBuffer = null,
1355
+ attnSoftcap = 0,
1356
+ slidingWindow = 0,
1357
+ kvLenBuffer = null,
1358
+ kvStart = 0,
1359
+ kvLayout = 'contiguous',
1360
+ kvPageTable = null,
1361
+ kvPageSize = 0,
1362
+ } = options;
1363
+
1364
+ if (kvLayout !== 'contiguous') {
1365
+ throw new Error(`[FlashAttention] kvLayout must be "contiguous", got "${kvLayout}".`);
1366
+ }
1367
+
1368
+ const device = execution.device;
1369
+ const numQueryBlocks = Math.max(1, Math.ceil(seqLen / FLASH_BLOCK_SIZE));
1370
+ const numKvSplits = chooseFlashNumKvSplits(kvLen);
1371
+ const singleSplit = numKvSplits === 1;
1372
+
1373
+ // Final output buffer (always allocated; bound to slot 4 on the single-split
1374
+ // fast path where the kernel writes normalised output directly).
1375
+ const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1376
+ const outputSize = seqLen * paddedHiddenSize * 4;
1377
+ const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_flash_output');
1378
+
1379
+ // Intermediate buffers for the multi-split path. On the single-split fast
1380
+ // path we bypass the reduce pass entirely and bind the output buffer in
1381
+ // slot 4 (partial_acc slot) while m/l bindings get tiny stub buffers — the
1382
+ // kernel skips writes to them.
1383
+ const partialAccBytes = singleSplit
1384
+ ? 4
1385
+ : numQueryBlocks * numHeads * numKvSplits * FLASH_BLOCK_SIZE * FLASH_HEAD_DIM * 4;
1386
+ const partialStatsBytes = singleSplit
1387
+ ? 4
1388
+ : numQueryBlocks * numHeads * numKvSplits * FLASH_BLOCK_SIZE * 4;
1389
+ const partialAcc = singleSplit
1390
+ ? outputBuf
1391
+ : acquireBuffer(partialAccBytes, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST, 'flash_partial_acc');
1392
+ const partialM = acquireBuffer(
1393
+ partialStatsBytes,
1394
+ GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1395
+ 'flash_partial_m'
1396
+ );
1397
+ const partialL = acquireBuffer(
1398
+ partialStatsBytes,
1399
+ GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
1400
+ 'flash_partial_l'
1401
+ );
1402
+
1403
+ // Pass 1 uniforms + dispatch.
1404
+ const flashUniform = createFlashAttentionUniformBuffer(device, execution.recorder, {
1405
+ numHeads,
1406
+ numKVHeads,
1407
+ headDim,
1408
+ kvLen,
1409
+ seqLen,
1410
+ scale,
1411
+ causal,
1412
+ startPos,
1413
+ attnSoftcap,
1414
+ slidingWindow,
1415
+ kvLenSource: kvLenBuffer ? 1 : 0,
1416
+ kvStart,
1417
+ pageSize: kvPageSize,
1418
+ kvLayout: 0, // contiguous only for now
1419
+ numKvSplits,
1420
+ });
1421
+
1422
+ const flashKernel = getFlashPrefillKernel(device);
1423
+ const flashPipeline = await flashKernel.getPipeline('prefill_flash_head256_f16kv');
1424
+ const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(device);
1425
+ const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(device);
1426
+
1427
+ const flashBindGroup = device.createBindGroup({
1428
+ label: 'attention_flash_prefill_bg',
1429
+ layout: flashPipeline.getBindGroupLayout(0),
1430
+ entries: [
1431
+ { binding: 0, resource: { buffer: flashUniform } },
1432
+ { binding: 1, resource: { buffer: Q.buffer } },
1433
+ { binding: 2, resource: { buffer: K.buffer } },
1434
+ { binding: 3, resource: { buffer: V.buffer } },
1435
+ { binding: 4, resource: { buffer: partialAcc } },
1436
+ { binding: 5, resource: { buffer: partialM } },
1437
+ { binding: 6, resource: { buffer: partialL } },
1438
+ { binding: 7, resource: { buffer: kvLenBinding } },
1439
+ { binding: 8, resource: { buffer: pageTableBinding } },
1440
+ ],
1441
+ });
1442
+
1443
+ dispatchAttentionKernel(
1444
+ execution,
1445
+ flashKernel,
1446
+ flashPipeline,
1447
+ flashBindGroup,
1448
+ numQueryBlocks * numHeads * numKvSplits
1449
+ );
1450
+ releaseAttentionUniform(execution, flashUniform);
1451
+
1452
+ // Pass 2 — reduce. Skipped on the single-split fast path where pass 1
1453
+ // already wrote the final normalised output directly to outputBuf.
1454
+ if (!singleSplit) {
1455
+ const reduceUniform = createFlashReduceUniformBuffer(device, execution.recorder, {
1456
+ numHeads,
1457
+ queryLen: seqLen,
1458
+ numKvSplits,
1459
+ });
1460
+
1461
+ const reduceKernel = getFlashReduceKernel(device);
1462
+ const reducePipeline = await reduceKernel.getPipeline('prefill_flash_reduce');
1463
+
1464
+ const reduceBindGroup = device.createBindGroup({
1465
+ label: 'attention_flash_reduce_bg',
1466
+ layout: reducePipeline.getBindGroupLayout(0),
1467
+ entries: [
1468
+ { binding: 0, resource: { buffer: reduceUniform } },
1469
+ { binding: 1, resource: { buffer: partialAcc } },
1470
+ { binding: 2, resource: { buffer: partialM } },
1471
+ { binding: 3, resource: { buffer: partialL } },
1472
+ { binding: 4, resource: { buffer: outputBuf } },
1473
+ ],
1474
+ });
1475
+
1476
+ const totalQh = seqLen * numHeads;
1477
+ const reduceWgX = Math.ceil(totalQh / FLASH_REDUCE_WG);
1478
+ dispatchAttentionKernel(
1479
+ execution,
1480
+ reduceKernel,
1481
+ reducePipeline,
1482
+ reduceBindGroup,
1483
+ [reduceWgX, FLASH_HEAD_DIM_VECS, 1]
1484
+ );
1485
+ releaseAttentionUniform(execution, reduceUniform);
1486
+ }
1487
+
1488
+ // Release intermediate buffers via the recorder's deferred cleanup so GPU
1489
+ // work completes before they re-enter the pool. On the single-split path
1490
+ // partialAcc IS the output buffer, so we skip it.
1491
+ const intermediates = singleSplit ? [partialM, partialL] : [partialAcc, partialM, partialL];
1492
+ if (execution.recorder) {
1493
+ for (const buf of intermediates) {
1494
+ execution.recorder.trackTemporaryBuffer(buf);
1495
+ }
1496
+ } else {
1497
+ device.queue.onSubmittedWorkDone().then(() => {
1498
+ for (const buf of intermediates) {
1499
+ releaseBuffer(buf);
1500
+ }
1501
+ });
1502
+ }
1503
+
1504
+ return createTensor(outputBuf, 'f32', [seqLen, numHeads, headDim], 'attention_flash_output');
1505
+ }
1506
+
1507
+ // Single-pass flash-attention dispatcher (ORT-style). 7-binding contract —
1508
+ // same as attention_head256_f16kv. One kernel launch, no reduce pass.
1509
+ // Workgroups: (num_heads, ceil(seqLen / ORT_FLASH_WG), 1) with ORT_FLASH_WG=64.
1510
+ const ORT_FLASH_WG = 64;
1511
+
1512
+ async function executeOrtFlashAttentionPrefill(recorder, Q, K, V, numHeads, headDim, options = {}) {
1513
+ if (headDim !== FLASH_HEAD_DIM) {
1514
+ throw new Error(`[OrtFlashAttention] headDim must be ${FLASH_HEAD_DIM}, got ${headDim}.`);
1515
+ }
1516
+ const execution = resolveAttentionExecution(recorder);
1517
+ const {
1518
+ seqLen = 1,
1519
+ kvLen = seqLen,
1520
+ numKVHeads = numHeads,
1521
+ scale = 1.0 / Math.sqrt(headDim),
1522
+ causal = true,
1523
+ startPos = 0,
1524
+ outputBuffer = null,
1525
+ attnSoftcap = 0,
1526
+ slidingWindow = 0,
1527
+ kvLenBuffer = null,
1528
+ kvStart = 0,
1529
+ kvLayout = 'contiguous',
1530
+ kvPageTable = null,
1531
+ kvPageSize = 0,
1532
+ } = options;
1533
+
1534
+ if (kvLayout !== 'contiguous') {
1535
+ throw new Error(`[OrtFlashAttention] kvLayout must be "contiguous", got "${kvLayout}".`);
1536
+ }
1537
+
1538
+ const device = execution.device;
1539
+ const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1540
+ const outputSize = seqLen * paddedHiddenSize * 4;
1541
+ const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_ort_flash_output');
1542
+
1543
+ const uniform = createAttentionUniformBuffer(device, execution.recorder, {
1544
+ numHeads,
1545
+ numKVHeads,
1546
+ headDim,
1547
+ kvLen,
1548
+ seqLen,
1549
+ scale,
1550
+ causal,
1551
+ startPos,
1552
+ attnSoftcap,
1553
+ slidingWindow,
1554
+ kvLenSource: kvLenBuffer ? 1 : 0,
1555
+ kvStart,
1556
+ pageSize: kvPageSize,
1557
+ kvLayout: 0, // contiguous only
1558
+ });
1559
+
1560
+ const kernel = new AttentionKernel(device);
1561
+ const pipeline = await kernel.getPipeline('prefill_flash_ort_head256_f16kv');
1562
+ const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(device);
1563
+ const pageTableBinding = kvPageTable || getPageTableFallbackBuffer(device);
1564
+
1565
+ const bindGroup = device.createBindGroup({
1566
+ label: 'attention_ort_flash_prefill_bg',
1567
+ layout: pipeline.getBindGroupLayout(0),
1568
+ entries: [
1569
+ { binding: 0, resource: { buffer: uniform } },
1570
+ { binding: 1, resource: { buffer: Q.buffer } },
1571
+ { binding: 2, resource: { buffer: K.buffer } },
1572
+ { binding: 3, resource: { buffer: V.buffer } },
1573
+ { binding: 4, resource: { buffer: outputBuf } },
1574
+ { binding: 5, resource: { buffer: kvLenBinding } },
1575
+ { binding: 6, resource: { buffer: pageTableBinding } },
1576
+ ],
1577
+ });
1578
+
1579
+ const numSeqTiles = Math.max(1, Math.ceil(seqLen / ORT_FLASH_WG));
1580
+ const workgroups = [numHeads, numSeqTiles, 1];
1581
+
1582
+ if (execution.recorder) {
1583
+ kernel.record(execution.recorder, pipeline, bindGroup, workgroups);
1584
+ } else {
1585
+ kernel.dispatch(pipeline, bindGroup, workgroups);
1586
+ }
1587
+
1588
+ if (uniform) releaseUniformBuffer(uniform);
1589
+ return createTensor(outputBuf, 'f32', [seqLen, numHeads, headDim], 'attention_ort_flash_output');
1590
+ }
1591
+
1158
1592
  async function executeAttentionTiered(
1159
1593
  recorder,
1160
1594
  Q,
@@ -14,7 +14,6 @@ const MAX_HEAD_DIM: u32 = 256u;
14
14
  override WORKGROUP_SIZE: u32 = 256u;
15
15
  override BIT_WIDTH: u32 = 4u;
16
16
  override PACK_FACTOR: u32 = 8u;
17
- override NUM_CENTROIDS: u32 = 16u;
18
17
 
19
18
  struct Uniforms {
20
19
  num_heads: u32,
@@ -12,7 +12,6 @@ const MAX_HEAD_DIM: u32 = 256u;
12
12
  override WORKGROUP_SIZE: u32 = 256u;
13
13
  override BIT_WIDTH_MSE: u32 = 3u;
14
14
  override PACK_FACTOR_MSE: u32 = 10u;
15
- override NUM_CENTROIDS_MSE: u32 = 8u;
16
15
 
17
16
  struct Uniforms {
18
17
  num_heads: u32,
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_decode.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused Multi-Head Attention Kernel (decode, f16 QKV + f16 output)
4
4
  //
5
5
  // Same as attention_decode_f16kv.wgsl but Q/K/V are f16 and output is f16.
@@ -1,5 +1,5 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/attention_decode.wgsl.
2
- // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:generate`.
2
+ // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
3
  // Fused Multi-Head Attention Kernel (f16 KV)
4
4
  //
5
5
  // Same as attention.wgsl but K/V are stored as f16 for KV-cache compression.