doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -43,6 +43,14 @@ function getTieredQuantMaxKVLen() {
43
43
  );
44
44
  }
45
45
 
46
+ function getContiguousQuantMaxKVLen() {
47
+ return getRequiredVariantMaxKVLen(
48
+ 'attention_contiguous_quant',
49
+ 'decode_contiguous_turboquant_f16kv',
50
+ 'attention_contiguous_quant.decode_contiguous_turboquant_f16kv'
51
+ );
52
+ }
53
+
46
54
 
47
55
  let kvLenFallbackBuffer = null;
48
56
  let kvLenFallbackBufferEpoch = -1;
@@ -161,6 +169,30 @@ class AttentionTieredQuantKernel extends KernelBase {
161
169
  }
162
170
  }
163
171
 
172
+ class AttentionContiguousQuantKernel extends KernelBase {
173
+
174
+ async getPipeline(variant) {
175
+ return this.getPipelineFor('attention_contiguous_quant', variant);
176
+ }
177
+
178
+ dispatch(
179
+ pipeline,
180
+ bindGroup,
181
+ workgroups
182
+ ) {
183
+ this.dispatchKernel(pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
184
+ }
185
+
186
+ record(
187
+ recorder,
188
+ pipeline,
189
+ bindGroup,
190
+ workgroups
191
+ ) {
192
+ this.recordKernel(recorder, pipeline, bindGroup, workgroups, 'attention_contiguous_quant');
193
+ }
194
+ }
195
+
164
196
  class AttentionBDPAKernel extends KernelBase {
165
197
  async getPipeline(variant) {
166
198
  return this.getPipelineFor('attention_bdpa', variant);
@@ -426,11 +458,12 @@ function validateAttentionVariant(
426
458
  }
427
459
 
428
460
  if (normalized.startsWith('decode_online')) {
461
+ const maxHeadDim = config.variantMetadata?.maxHeadDim ?? thresholds.subgroupMaxHeadDim;
429
462
  if (!caps.hasSubgroups) {
430
463
  throw new Error(`Attention kernel "${variant}" requires subgroup support.`);
431
464
  }
432
- if (headDim > thresholds.subgroupMaxHeadDim) {
433
- throw new Error(`Attention kernel "${variant}" requires headDim <= ${thresholds.subgroupMaxHeadDim} but got ${headDim}.`);
465
+ if (headDim > maxHeadDim) {
466
+ throw new Error(`Attention kernel "${variant}" requires headDim <= ${maxHeadDim} but got ${headDim}.`);
434
467
  }
435
468
  if (sharedLimit < thresholds.subgroupShared) {
436
469
  throw new Error(`Attention kernel "${variant}" requires shared >= ${thresholds.subgroupShared} but got ${sharedLimit}.`);
@@ -445,10 +478,21 @@ function validateAttentionVariant(
445
478
  && !normalized.startsWith('decode_online')
446
479
  && !isChunked;
447
480
  if (isTiled) {
448
- const requiredShared = isSmall
449
- ? (useF16KV ? thresholds.smallSharedF16 : thresholds.smallSharedF32)
450
- : (useF16KV ? thresholds.largeSharedF16 : thresholds.largeSharedF32);
451
- const maxHeadDim = isSmall ? thresholds.smallMaxHeadDim : thresholds.largeMaxHeadDim;
481
+ const metadata = config.variantMetadata ?? {};
482
+ const requiredShared = metadata.requiredShared ?? (
483
+ isSmall
484
+ ? (useF16KV ? thresholds.smallSharedF16 : thresholds.smallSharedF32)
485
+ : (useF16KV ? thresholds.largeSharedF16 : thresholds.largeSharedF32)
486
+ );
487
+ const maxHeadDim = metadata.maxHeadDim ?? (isSmall ? thresholds.smallMaxHeadDim : thresholds.largeMaxHeadDim);
488
+ const minHeadDim = metadata.minHeadDim ?? 0;
489
+ const exactHeadDim = metadata.exactHeadDim;
490
+ if (Number.isFinite(exactHeadDim) && headDim !== exactHeadDim) {
491
+ throw new Error(`Attention kernel "${variant}" requires headDim == ${exactHeadDim} but got ${headDim}.`);
492
+ }
493
+ if (headDim < minHeadDim) {
494
+ throw new Error(`Attention kernel "${variant}" requires headDim >= ${minHeadDim} but got ${headDim}.`);
495
+ }
452
496
  if (headDim > maxHeadDim) {
453
497
  throw new Error(`Attention kernel "${variant}" requires headDim <= ${maxHeadDim} but got ${headDim}.`);
454
498
  }
@@ -637,7 +681,7 @@ function createAttentionUniformBuffer(
637
681
  ) {
638
682
  return createUniformBufferWithView(
639
683
  'attention_uniforms',
640
- 64, // 60 bytes used + 4 padding for 16-byte alignment
684
+ 80,
641
685
  (view) => {
642
686
  view.setUint32(0, params.numHeads, true);
643
687
  view.setUint32(4, params.numKVHeads, true);
@@ -653,7 +697,11 @@ function createAttentionUniformBuffer(
653
697
  view.setUint32(44, params.kvStart ?? 0, true);
654
698
  view.setUint32(48, params.pageSize ?? 0, true);
655
699
  view.setUint32(52, params.kvLayout ?? 0, true);
656
- view.setUint32(56, 0, true);
700
+ view.setUint32(56, params.bidirectionalSpanStart ?? 0, true);
701
+ view.setUint32(60, params.bidirectionalSpanLength ?? 0, true);
702
+ view.setUint32(64, 0, true);
703
+ view.setUint32(68, 0, true);
704
+ view.setUint32(72, 0, true);
657
705
  },
658
706
  recorder,
659
707
  device
@@ -722,6 +770,44 @@ function createTieredQuantAttentionUniformBuffer(
722
770
  );
723
771
  }
724
772
 
773
+ function createContiguousQuantAttentionUniformBuffer(
774
+ device,
775
+ recorder,
776
+ params
777
+ ) {
778
+ const hasProdFields = params.packedStrideMSE != null;
779
+ const size = hasProdFields ? 64 : 48;
780
+ return createUniformBufferWithView(
781
+ 'attention_contiguous_quant_uniforms',
782
+ size,
783
+ (view) => {
784
+ view.setUint32(0, params.numHeads, true);
785
+ view.setUint32(4, params.numKVHeads, true);
786
+ view.setUint32(8, params.headDim, true);
787
+ view.setUint32(12, params.kvLen, true);
788
+ view.setUint32(16, params.seqLen, true);
789
+ view.setFloat32(20, params.scale, true);
790
+ view.setUint32(24, params.causal ? 1 : 0, true);
791
+ view.setUint32(28, params.startPos, true);
792
+ view.setFloat32(32, params.attnSoftcap, true);
793
+ view.setUint32(36, params.slidingWindow, true);
794
+ if (hasProdFields) {
795
+ view.setUint32(40, params.packedStrideMSE, true);
796
+ view.setUint32(44, params.packedStrideResidual, true);
797
+ view.setUint32(48, 0, true);
798
+ view.setUint32(52, 0, true);
799
+ view.setUint32(56, 0, true);
800
+ view.setUint32(60, 0, true);
801
+ } else {
802
+ view.setUint32(40, params.packedStride, true);
803
+ view.setUint32(44, 0, true);
804
+ }
805
+ },
806
+ recorder,
807
+ device
808
+ );
809
+ }
810
+
725
811
  function createBDPAAttentionUniformBuffer(
726
812
  device,
727
813
  recorder,
@@ -924,6 +1010,8 @@ async function executeAttention(
924
1010
  numKVHeads = numHeads,
925
1011
  scale = 1.0 / Math.sqrt(headDim),
926
1012
  causal = true,
1013
+ bidirectionalSpanStart = 0,
1014
+ bidirectionalSpanLength = 0,
927
1015
  startPos = 0,
928
1016
  layerIdx,
929
1017
  outputBuffer = null,
@@ -938,6 +1026,18 @@ async function executeAttention(
938
1026
  kvPageSize = 0,
939
1027
  kernelPath = null,
940
1028
  } = options;
1029
+ if (!Number.isFinite(bidirectionalSpanStart) || Math.floor(bidirectionalSpanStart) !== bidirectionalSpanStart || bidirectionalSpanStart < 0) {
1030
+ throw new Error(`Attention bidirectionalSpanStart must be a non-negative integer, got ${bidirectionalSpanStart}.`);
1031
+ }
1032
+ if (!Number.isFinite(bidirectionalSpanLength) || Math.floor(bidirectionalSpanLength) !== bidirectionalSpanLength || bidirectionalSpanLength < 0) {
1033
+ throw new Error(`Attention bidirectionalSpanLength must be a non-negative integer, got ${bidirectionalSpanLength}.`);
1034
+ }
1035
+ if (bidirectionalSpanLength > 0 && (bidirectionalSpanStart + bidirectionalSpanLength) > (kvStart + kvLen)) {
1036
+ throw new Error(
1037
+ `Attention bidirectional span [${bidirectionalSpanStart}, ${bidirectionalSpanStart + bidirectionalSpanLength}) ` +
1038
+ `exceeds KV extent [${kvStart}, ${kvStart + kvLen}).`
1039
+ );
1040
+ }
941
1041
 
942
1042
  const limits = getDeviceLimits();
943
1043
  const sharedLimit = limits?.maxComputeWorkgroupStorageSize ?? Infinity;
@@ -995,6 +1095,8 @@ async function executeAttention(
995
1095
  kvStart,
996
1096
  pageSize: kvPageSize,
997
1097
  kvLayout: kvLayout === 'paged' ? 2 : (kvLayout === 'ring' ? 1 : 0),
1098
+ bidirectionalSpanStart,
1099
+ bidirectionalSpanLength,
998
1100
  });
999
1101
 
1000
1102
  const kvLenBinding = kvLenBuffer || getKvLenFallbackBuffer(execution.device);
@@ -1209,8 +1311,26 @@ async function executeAttentionTieredQuant(
1209
1311
  hotStart = 0,
1210
1312
  packedStride = 0,
1211
1313
  mode = 'int8',
1314
+ // TurboQuant additional buffers
1315
+ rotationMatrixBuffer = null,
1316
+ codebookCentroidsBuffer = null,
1317
+ residualKBuffer = null,
1318
+ residualVBuffer = null,
1319
+ residualNormsKBuffer = null,
1320
+ residualNormsVBuffer = null,
1321
+ qjlMatrixBuffer = null,
1212
1322
  } = options;
1213
1323
 
1324
+ if (mode === 'turboquant_outlier') {
1325
+ throw new Error(
1326
+ 'TurboQuant outlier attention is not supported yet. ' +
1327
+ 'Outlier-mode decode kernels are not wired end to end.'
1328
+ );
1329
+ }
1330
+
1331
+ const isTurboQuant = mode === 'turboquant' || mode === 'turboquant_prod';
1332
+ const isProd = mode === 'turboquant_prod';
1333
+
1214
1334
  const totalLen = coldLen + hotLen;
1215
1335
  const maxKVLen = getTieredQuantMaxKVLen();
1216
1336
  if (totalLen > maxKVLen) {
@@ -1224,6 +1344,16 @@ async function executeAttentionTieredQuant(
1224
1344
  throw new Error('Tiered quant attention requires f32 Q.');
1225
1345
  }
1226
1346
 
1347
+ if (isTurboQuant && !rotationMatrixBuffer) {
1348
+ throw new Error('TurboQuant tiered quant attention requires rotationMatrixBuffer.');
1349
+ }
1350
+ if (isTurboQuant && !codebookCentroidsBuffer) {
1351
+ throw new Error('TurboQuant tiered quant attention requires codebookCentroidsBuffer.');
1352
+ }
1353
+ if (isProd && !qjlMatrixBuffer) {
1354
+ throw new Error('TurboQuant prod tiered quant attention requires qjlMatrixBuffer.');
1355
+ }
1356
+
1227
1357
  const variant = selectKernelRuleValue('attention', 'tieredQuantVariant', { mode });
1228
1358
  const caps = getKernelCapabilities();
1229
1359
  const config = getKernelConfig('attention_tiered_quant', variant);
@@ -1277,11 +1407,33 @@ async function executeAttentionTieredQuant(
1277
1407
  assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 5, 'coldPackedV', coldPackedV);
1278
1408
  assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 6, 'coldScalesK', coldScalesK);
1279
1409
  assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 7, 'coldScalesV', coldScalesV);
1280
- assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 8, 'output', outputBuf);
1281
- const bindGroup = execution.device.createBindGroup({
1282
- label: 'attention_tiered_quant_bind_group',
1283
- layout: pipeline.getBindGroupLayout(0),
1284
- entries: [
1410
+
1411
+ let entries;
1412
+ if (isProd) {
1413
+ // TurboQuant prod tiered: 16 bindings
1414
+ assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 8, 'residual_k', residualKBuffer);
1415
+ assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 15, 'qjl_matrix', qjlMatrixBuffer);
1416
+ entries = [
1417
+ { binding: 0, resource: { buffer: uniformBuffer } },
1418
+ { binding: 1, resource: { buffer: Q.buffer } },
1419
+ { binding: 2, resource: { buffer: hotK.buffer } },
1420
+ { binding: 3, resource: { buffer: hotV.buffer } },
1421
+ { binding: 4, resource: { buffer: coldPackedK } },
1422
+ { binding: 5, resource: { buffer: coldPackedV } },
1423
+ { binding: 6, resource: { buffer: coldScalesK } },
1424
+ { binding: 7, resource: { buffer: coldScalesV } },
1425
+ { binding: 8, resource: { buffer: residualKBuffer } },
1426
+ { binding: 9, resource: { buffer: residualVBuffer } },
1427
+ { binding: 10, resource: { buffer: residualNormsKBuffer } },
1428
+ { binding: 11, resource: { buffer: residualNormsVBuffer } },
1429
+ { binding: 12, resource: { buffer: outputBuf } },
1430
+ { binding: 13, resource: { buffer: rotationMatrixBuffer } },
1431
+ { binding: 14, resource: { buffer: codebookCentroidsBuffer } },
1432
+ { binding: 15, resource: { buffer: qjlMatrixBuffer } },
1433
+ ];
1434
+ } else if (isTurboQuant) {
1435
+ // TurboQuant MSE tiered: 11 bindings
1436
+ entries = [
1285
1437
  { binding: 0, resource: { buffer: uniformBuffer } },
1286
1438
  { binding: 1, resource: { buffer: Q.buffer } },
1287
1439
  { binding: 2, resource: { buffer: hotK.buffer } },
@@ -1291,7 +1443,29 @@ async function executeAttentionTieredQuant(
1291
1443
  { binding: 6, resource: { buffer: coldScalesK } },
1292
1444
  { binding: 7, resource: { buffer: coldScalesV } },
1293
1445
  { binding: 8, resource: { buffer: outputBuf } },
1294
- ],
1446
+ { binding: 9, resource: { buffer: rotationMatrixBuffer } },
1447
+ { binding: 10, resource: { buffer: codebookCentroidsBuffer } },
1448
+ ];
1449
+ } else {
1450
+ // Standard int4/int8: 9 bindings
1451
+ assertAttentionBindGroupBuffer('attention_tiered_quant', variant, 8, 'output', outputBuf);
1452
+ entries = [
1453
+ { binding: 0, resource: { buffer: uniformBuffer } },
1454
+ { binding: 1, resource: { buffer: Q.buffer } },
1455
+ { binding: 2, resource: { buffer: hotK.buffer } },
1456
+ { binding: 3, resource: { buffer: hotV.buffer } },
1457
+ { binding: 4, resource: { buffer: coldPackedK } },
1458
+ { binding: 5, resource: { buffer: coldPackedV } },
1459
+ { binding: 6, resource: { buffer: coldScalesK } },
1460
+ { binding: 7, resource: { buffer: coldScalesV } },
1461
+ { binding: 8, resource: { buffer: outputBuf } },
1462
+ ];
1463
+ }
1464
+
1465
+ const bindGroup = execution.device.createBindGroup({
1466
+ label: 'attention_tiered_quant_bind_group',
1467
+ layout: pipeline.getBindGroupLayout(0),
1468
+ entries,
1295
1469
  });
1296
1470
 
1297
1471
  dispatchAttentionKernel(execution, kernel, pipeline, bindGroup, numHeads);
@@ -1435,3 +1609,174 @@ export async function recordAttentionTieredQuant(
1435
1609
  options
1436
1610
  );
1437
1611
  }
1612
+
1613
+
1614
+ // =============================================================================
1615
+ // Contiguous Quantized Attention (TurboQuant for full-attention models)
1616
+ // =============================================================================
1617
+
1618
+ async function executeAttentionContiguousQuant(
1619
+ recorder,
1620
+ Q,
1621
+ packedK,
1622
+ packedV,
1623
+ scalesK,
1624
+ scalesV,
1625
+ numHeads,
1626
+ headDim,
1627
+ options = {}
1628
+ ) {
1629
+ const execution = resolveAttentionExecution(recorder);
1630
+ const {
1631
+ seqLen = 1,
1632
+ kvLen = 0,
1633
+ numKVHeads = numHeads,
1634
+ scale = 1.0 / Math.sqrt(headDim),
1635
+ causal = true,
1636
+ startPos = 0,
1637
+ outputBuffer = null,
1638
+ attnSoftcap = 0,
1639
+ slidingWindow = 0,
1640
+ packedStride = 0,
1641
+ mode = 'turboquant',
1642
+ rotationMatrixBuffer = null,
1643
+ codebookCentroidsBuffer = null,
1644
+ // Prod-mode additional buffers
1645
+ residualKBuffer = null,
1646
+ residualVBuffer = null,
1647
+ residualNormsKBuffer = null,
1648
+ residualNormsVBuffer = null,
1649
+ qjlMatrixBuffer = null,
1650
+ packedStrideMSE = 0,
1651
+ packedStrideResidual = 0,
1652
+ } = options;
1653
+
1654
+ const maxKVLen = getContiguousQuantMaxKVLen();
1655
+ if (kvLen > maxKVLen) {
1656
+ throw new Error(`Contiguous quant attention requires kvLen <= ${maxKVLen} but got ${kvLen}.`);
1657
+ }
1658
+ if (!Number.isFinite(packedStride) || packedStride <= 0) {
1659
+ throw new Error('Contiguous quant attention requires packedStride > 0.');
1660
+ }
1661
+ if (Q.dtype !== 'f32') {
1662
+ throw new Error('Contiguous quant attention requires f32 Q.');
1663
+ }
1664
+ if (!rotationMatrixBuffer) {
1665
+ throw new Error('Contiguous quant attention requires rotationMatrixBuffer.');
1666
+ }
1667
+ if (!codebookCentroidsBuffer) {
1668
+ throw new Error('Contiguous quant attention requires codebookCentroidsBuffer.');
1669
+ }
1670
+
1671
+ const isProd = mode === 'turboquant_prod';
1672
+ const variant = selectKernelRuleValue('attention', 'contiguousQuantVariant', { mode });
1673
+ const caps = getKernelCapabilities();
1674
+ const config = getKernelConfig('attention_contiguous_quant', variant);
1675
+ if (!hasRequiredFeatures(config.requires, caps)) {
1676
+ throw new Error(`Contiguous quant attention kernel "${variant}" requires unsupported GPU features.`);
1677
+ }
1678
+
1679
+ const kernel = new AttentionContiguousQuantKernel(execution.device);
1680
+ const pipeline = await kernel.getPipeline(variant);
1681
+
1682
+ const outputDtype = config.outputDtype;
1683
+ if (!outputDtype) {
1684
+ throw new Error(`Kernel config missing outputDtype for attention_contiguous_quant variant "${variant}".`);
1685
+ }
1686
+ const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
1687
+ const paddedHiddenSize = padToQ4KBlock(numHeads * headDim);
1688
+ const outputSize = seqLen * paddedHiddenSize * bytesPerElement;
1689
+ const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'attention_contiguous_quant_output');
1690
+
1691
+ const uniformParams = isProd
1692
+ ? {
1693
+ numHeads, numKVHeads, headDim, kvLen, seqLen, scale, causal, startPos,
1694
+ attnSoftcap, slidingWindow, packedStrideMSE, packedStrideResidual,
1695
+ }
1696
+ : {
1697
+ numHeads, numKVHeads, headDim, kvLen, seqLen, scale, causal, startPos,
1698
+ attnSoftcap, slidingWindow, packedStride,
1699
+ };
1700
+ const uniformBuffer = createContiguousQuantAttentionUniformBuffer(
1701
+ execution.device, execution.recorder, uniformParams
1702
+ );
1703
+
1704
+ let entries;
1705
+ if (isProd) {
1706
+ // Contiguous prod: 14 bindings
1707
+ assertAttentionBindGroupBuffer('attention_contiguous_quant', variant, 6, 'residual_k', residualKBuffer);
1708
+ assertAttentionBindGroupBuffer('attention_contiguous_quant', variant, 7, 'residual_v', residualVBuffer);
1709
+ assertAttentionBindGroupBuffer('attention_contiguous_quant', variant, 13, 'qjl_matrix', qjlMatrixBuffer);
1710
+ entries = [
1711
+ { binding: 0, resource: { buffer: uniformBuffer } },
1712
+ { binding: 1, resource: { buffer: Q.buffer } },
1713
+ { binding: 2, resource: { buffer: packedK } },
1714
+ { binding: 3, resource: { buffer: packedV } },
1715
+ { binding: 4, resource: { buffer: scalesK } },
1716
+ { binding: 5, resource: { buffer: scalesV } },
1717
+ { binding: 6, resource: { buffer: residualKBuffer } },
1718
+ { binding: 7, resource: { buffer: residualVBuffer } },
1719
+ { binding: 8, resource: { buffer: residualNormsKBuffer } },
1720
+ { binding: 9, resource: { buffer: residualNormsVBuffer } },
1721
+ { binding: 10, resource: { buffer: outputBuf } },
1722
+ { binding: 11, resource: { buffer: rotationMatrixBuffer } },
1723
+ { binding: 12, resource: { buffer: codebookCentroidsBuffer } },
1724
+ { binding: 13, resource: { buffer: qjlMatrixBuffer } },
1725
+ ];
1726
+ } else {
1727
+ // Contiguous MSE: 9 bindings
1728
+ entries = [
1729
+ { binding: 0, resource: { buffer: uniformBuffer } },
1730
+ { binding: 1, resource: { buffer: Q.buffer } },
1731
+ { binding: 2, resource: { buffer: packedK } },
1732
+ { binding: 3, resource: { buffer: packedV } },
1733
+ { binding: 4, resource: { buffer: scalesK } },
1734
+ { binding: 5, resource: { buffer: scalesV } },
1735
+ { binding: 6, resource: { buffer: outputBuf } },
1736
+ { binding: 7, resource: { buffer: rotationMatrixBuffer } },
1737
+ { binding: 8, resource: { buffer: codebookCentroidsBuffer } },
1738
+ ];
1739
+ }
1740
+
1741
+ const bindGroup = execution.device.createBindGroup({
1742
+ label: 'attention_contiguous_quant_bind_group',
1743
+ layout: pipeline.getBindGroupLayout(0),
1744
+ entries,
1745
+ });
1746
+
1747
+ dispatchAttentionKernel(execution, kernel, pipeline, bindGroup, numHeads);
1748
+ releaseAttentionUniform(execution, uniformBuffer);
1749
+
1750
+ return createTensor(outputBuf, outputDtype, [seqLen, numHeads, headDim], 'attention_contiguous_quant_output');
1751
+ }
1752
+
1753
+ export async function runAttentionContiguousQuant(
1754
+ Q,
1755
+ packedK,
1756
+ packedV,
1757
+ scalesK,
1758
+ scalesV,
1759
+ numHeads,
1760
+ headDim,
1761
+ options = {}
1762
+ ) {
1763
+ return executeAttentionContiguousQuant(
1764
+ null, Q, packedK, packedV, scalesK, scalesV, numHeads, headDim, options
1765
+ );
1766
+ }
1767
+
1768
+ export async function recordAttentionContiguousQuant(
1769
+ recorder,
1770
+ Q,
1771
+ packedK,
1772
+ packedV,
1773
+ scalesK,
1774
+ scalesV,
1775
+ numHeads,
1776
+ headDim,
1777
+ options = {}
1778
+ ) {
1779
+ return executeAttentionContiguousQuant(
1780
+ recorder, Q, packedK, packedV, scalesK, scalesV, numHeads, headDim, options
1781
+ );
1782
+ }
@@ -30,7 +30,11 @@ struct Uniforms {
30
30
  kv_start: u32,
31
31
  page_size: u32,
32
32
  kv_layout: u32,
33
- _pad: u32,
33
+ bidirectional_span_start: u32,
34
+ bidirectional_span_length: u32,
35
+ _pad0: u32,
36
+ _pad1: u32,
37
+ _pad2: u32,
34
38
  }
35
39
 
36
40
  @group(0) @binding(0) var<uniform> u: Uniforms;
@@ -53,6 +57,18 @@ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
53
57
  return query_head_idx / heads_per_kv;
54
58
  }
55
59
 
60
+ fn is_bidirectional_span_visible(abs_query: u32, abs_key: u32) -> bool {
61
+ if (u.bidirectional_span_length == 0u) {
62
+ return false;
63
+ }
64
+ let span_start = u.bidirectional_span_start;
65
+ let span_end = span_start + u.bidirectional_span_length;
66
+ return abs_query >= span_start
67
+ && abs_query < span_end
68
+ && abs_key >= span_start
69
+ && abs_key < span_end;
70
+ }
71
+
56
72
  // Check if position should be masked (causal + sliding window attention)
57
73
  fn is_masked(query_pos: u32, key_pos: u32) -> bool {
58
74
  // Compute absolute positions
@@ -61,6 +77,9 @@ fn is_masked(query_pos: u32, key_pos: u32) -> bool {
61
77
 
62
78
  // Causal mask: query can only attend to keys at same or earlier positions
63
79
  if (u.is_causal != 0u && abs_key > abs_query) {
80
+ if (is_bidirectional_span_visible(abs_query, abs_key)) {
81
+ return false;
82
+ }
64
83
  return true;
65
84
  }
66
85
 
@@ -0,0 +1,180 @@
1
+ // Contiguous Decode Attention Kernel (TurboQuant MSE, f32 output)
2
+ //
3
+ // All KV entries are TurboQuant-quantized (no hot/cold split).
4
+ // For full-attention models (Qwen 3.5, LFM) that cannot use tiered layout.
5
+ // Q is rotated by Π, dot products computed in rotated domain,
6
+ // V accumulated in rotated domain then inverse-rotated.
7
+
8
+ enable f16;
9
+
10
+ const MAX_KV_LEN: u32 = 2048u;
11
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
12
+ const MAX_HEAD_DIM: u32 = 256u;
13
+
14
+ override WORKGROUP_SIZE: u32 = 256u;
15
+ override BIT_WIDTH: u32 = 4u;
16
+ override PACK_FACTOR: u32 = 8u;
17
+ override NUM_CENTROIDS: u32 = 16u;
18
+
19
+ struct Uniforms {
20
+ num_heads: u32,
21
+ num_kv_heads: u32,
22
+ head_dim: u32,
23
+ kv_len: u32,
24
+ seq_len: u32,
25
+ scale: f32,
26
+ is_causal: u32,
27
+ start_pos: u32,
28
+ attn_softcap: f32,
29
+ sliding_window: u32,
30
+ packed_stride: u32,
31
+ _pad: u32,
32
+ }
33
+
34
+ @group(0) @binding(0) var<uniform> u: Uniforms;
35
+ @group(0) @binding(1) var<storage, read> Q: array<f32>;
36
+ @group(0) @binding(2) var<storage, read> K_packed: array<u32>;
37
+ @group(0) @binding(3) var<storage, read> V_packed: array<u32>;
38
+ @group(0) @binding(4) var<storage, read> scales_k: array<f16>;
39
+ @group(0) @binding(5) var<storage, read> scales_v: array<f16>;
40
+ @group(0) @binding(6) var<storage, read_write> output: array<f32>;
41
+ @group(0) @binding(7) var<storage, read> rotation_matrix: array<f32>;
42
+ @group(0) @binding(8) var<storage, read> codebook_centroids: array<f32>;
43
+
44
+ var<workgroup> shared_scores: array<f32, MAX_KV_LEN>;
45
+ var<workgroup> shared_partial: array<f32, MAX_WORKGROUP_SIZE>;
46
+ var<workgroup> shared_max: f32;
47
+ var<workgroup> shared_sum: f32;
48
+ var<workgroup> shared_acc: array<f32, MAX_HEAD_DIM>;
49
+ var<workgroup> shared_rot_q: array<f32, MAX_HEAD_DIM>;
50
+
51
+ fn get_kv_head_idx(query_head_idx: u32) -> u32 {
52
+ return query_head_idx / (u.num_heads / u.num_kv_heads);
53
+ }
54
+
55
+ fn is_masked(abs_key: u32) -> bool {
56
+ let abs_query = u.start_pos;
57
+ if (u.is_causal != 0u && abs_key > abs_query) { return true; }
58
+ if (u.sliding_window > 0u && abs_query >= u.sliding_window) {
59
+ if (abs_key < abs_query - u.sliding_window + 1u) { return true; }
60
+ }
61
+ return false;
62
+ }
63
+
64
+ fn unpack_index(packed_val: u32, lane: u32) -> u32 {
65
+ return (packed_val >> (BIT_WIDTH * lane)) & ((1u << BIT_WIDTH) - 1u);
66
+ }
67
+
68
+ fn dequant_k(key_pos: u32, kv_head_idx: u32, tid: u32) -> f32 {
69
+ let pack_idx = tid / PACK_FACTOR;
70
+ if (pack_idx >= u.packed_stride) { return 0.0; }
71
+ let lane = tid - pack_idx * PACK_FACTOR;
72
+ let base = (key_pos * u.num_kv_heads + kv_head_idx) * u.packed_stride + pack_idx;
73
+ let idx = unpack_index(K_packed[base], lane);
74
+ let norm = f32(scales_k[key_pos * u.num_kv_heads + kv_head_idx]);
75
+ return codebook_centroids[idx] * norm;
76
+ }
77
+
78
+ fn dequant_v(key_pos: u32, kv_head_idx: u32, tid: u32) -> f32 {
79
+ let pack_idx = tid / PACK_FACTOR;
80
+ if (pack_idx >= u.packed_stride) { return 0.0; }
81
+ let lane = tid - pack_idx * PACK_FACTOR;
82
+ let base = (key_pos * u.num_kv_heads + kv_head_idx) * u.packed_stride + pack_idx;
83
+ let idx = unpack_index(V_packed[base], lane);
84
+ let norm = f32(scales_v[key_pos * u.num_kv_heads + kv_head_idx]);
85
+ return codebook_centroids[idx] * norm;
86
+ }
87
+
88
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
89
+ fn main(
90
+ @builtin(local_invocation_id) local_id: vec3<u32>,
91
+ @builtin(workgroup_id) workgroup_id: vec3<u32>,
92
+ ) {
93
+ let head_idx = workgroup_id.x;
94
+ let tid = local_id.x;
95
+ let head_dim = u.head_dim;
96
+ let kv_len = u.kv_len;
97
+ let scale = u.scale;
98
+ let kv_head_idx = get_kv_head_idx(head_idx);
99
+
100
+ if (WORKGROUP_SIZE > MAX_WORKGROUP_SIZE || head_dim > MAX_HEAD_DIM || kv_len > MAX_KV_LEN) {
101
+ return;
102
+ }
103
+
104
+ let valid = tid < head_dim;
105
+
106
+ // Rotate Q by Π for dot products in rotated domain
107
+ if (valid) {
108
+ var rot_q: f32 = 0.0;
109
+ let row_base = tid * head_dim;
110
+ for (var j: u32 = 0u; j < head_dim; j++) {
111
+ rot_q += rotation_matrix[row_base + j] * Q[head_idx * head_dim + j];
112
+ }
113
+ shared_rot_q[tid] = rot_q;
114
+ }
115
+ workgroupBarrier();
116
+
117
+ if (valid) { shared_acc[tid] = 0.0; }
118
+
119
+ // Score computation
120
+ for (var k_pos: u32 = 0u; k_pos < kv_len; k_pos++) {
121
+ var k_val: f32 = 0.0;
122
+ if (valid) {
123
+ k_val = dequant_k(k_pos, kv_head_idx, tid);
124
+ }
125
+
126
+ let partial = shared_rot_q[tid] * k_val;
127
+ shared_partial[tid] = select(0.0, partial, valid);
128
+ workgroupBarrier();
129
+
130
+ if (tid == 0u) {
131
+ var dot: f32 = 0.0;
132
+ for (var d: u32 = 0u; d < head_dim; d++) { dot += shared_partial[d]; }
133
+ var s = dot * scale;
134
+ if (u.attn_softcap > 0.0) { s = tanh(s / u.attn_softcap) * u.attn_softcap; }
135
+ if (is_masked(k_pos)) { s = -3.402823e+38; }
136
+ shared_scores[k_pos] = s;
137
+ }
138
+ workgroupBarrier();
139
+ }
140
+
141
+ // Softmax
142
+ if (tid == 0u) {
143
+ var m: f32 = -3.402823e+38;
144
+ for (var k: u32 = 0u; k < kv_len; k++) { m = max(m, shared_scores[k]); }
145
+ shared_max = m;
146
+ }
147
+ workgroupBarrier();
148
+
149
+ if (tid == 0u) {
150
+ var s: f32 = 0.0;
151
+ for (var k: u32 = 0u; k < kv_len; k++) {
152
+ let w = exp(shared_scores[k] - shared_max);
153
+ shared_scores[k] = w;
154
+ s += w;
155
+ }
156
+ shared_sum = s;
157
+ }
158
+ workgroupBarrier();
159
+
160
+ let inv_sum = select(0.0, 1.0 / shared_sum, shared_sum > 0.0);
161
+
162
+ // V accumulation in rotated domain
163
+ if (valid) {
164
+ var acc: f32 = 0.0;
165
+ for (var si: u32 = 0u; si < kv_len; si++) {
166
+ acc += shared_scores[si] * dequant_v(si, kv_head_idx, tid);
167
+ }
168
+ shared_acc[tid] = acc;
169
+ }
170
+ workgroupBarrier();
171
+
172
+ // Inverse rotation Π^T and normalize
173
+ if (valid) {
174
+ var inv_rot: f32 = 0.0;
175
+ for (var j: u32 = 0u; j < head_dim; j++) {
176
+ inv_rot += rotation_matrix[j * head_dim + tid] * shared_acc[j];
177
+ }
178
+ output[head_idx * head_dim + tid] = inv_rot * inv_sum;
179
+ }
180
+ }