doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -2,18 +2,23 @@ import { getDevice, getDeviceEpoch } from '../device.js';
2
2
  import { WORKGROUP_SIZES } from './constants.js';
3
3
  import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
4
4
  import { createTensor } from '../tensor.js';
5
+ import { castF32ToF16, recordCastF32ToF16 } from './cast.js';
5
6
  import {
6
7
  createUniformBufferFromData,
7
8
  getOrCreateBindGroupLayout,
8
9
  getOrCreatePipelineLayout,
9
10
  } from './utils.js';
10
11
  import { recordDispatch } from './dispatch.js';
12
+ import { selectRuleValue } from '../../rules/rule-registry.js';
13
+ import { assertImplicitDtypeTransitionAllowed } from '../../inference/pipelines/text/dtype-contract.js';
11
14
 
12
15
  const CONV_WORKGROUP_SIZE = WORKGROUP_SIZES.DEFAULT;
13
16
  const HEAD_WORKGROUP_SIZE = 128;
14
17
 
15
- const CONV_SHADER = /* wgsl */ `
16
- override WORKGROUP_SIZE: u32 = 256u;
18
+ function buildLinearAttentionConvShader(inputDtype) {
19
+ const inputScalar = selectRuleValue('shared', 'dtype', 'f16OrF32', { useF16: inputDtype === 'f16' });
20
+ const enableF16 = inputScalar === 'f16' ? 'enable f16;\n\n' : '';
21
+ return /* wgsl */ `${enableF16}override WORKGROUP_SIZE: u32 = 256u;
17
22
 
18
23
  struct LinearAttentionParams {
19
24
  num_tokens: u32,
@@ -35,7 +40,7 @@ struct LinearAttentionParams {
35
40
  }
36
41
 
37
42
  @group(0) @binding(0) var<uniform> params: LinearAttentionParams;
38
- @group(0) @binding(1) var<storage, read> qkv: array<f32>;
43
+ @group(0) @binding(1) var<storage, read> qkv: array<${inputScalar}>;
39
44
  @group(0) @binding(2) var<storage, read> conv_weight: array<f32>;
40
45
  @group(0) @binding(3) var<storage, read_write> conv_state: array<f32>;
41
46
  @group(0) @binding(4) var<storage, read_write> conv_out: array<f32>;
@@ -61,7 +66,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
61
66
 
62
67
  for (var token_idx: u32 = 0u; token_idx < params.num_tokens; token_idx = token_idx + 1u) {
63
68
  let qkv_idx = token_idx * params.conv_dim + channel;
64
- let newest = qkv[qkv_idx];
69
+ let newest = f32(qkv[qkv_idx]);
65
70
 
66
71
  for (var k: u32 = 0u; k + 1u < kernel_size; k = k + 1u) {
67
72
  conv_state[state_base + k] = conv_state[state_base + k + 1u];
@@ -77,9 +82,12 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
77
82
  }
78
83
  }
79
84
  `;
85
+ }
80
86
 
81
- const RECURRENT_SHADER = /* wgsl */ `
82
- override WORKGROUP_SIZE: u32 = 128u;
87
+ function buildLinearAttentionRecurrentShader(inputDtype) {
88
+ const inputScalar = selectRuleValue('shared', 'dtype', 'f16OrF32', { useF16: inputDtype === 'f16' });
89
+ const enableF16 = inputScalar === 'f16' ? 'enable f16;\n\n' : '';
90
+ return /* wgsl */ `${enableF16}override WORKGROUP_SIZE: u32 = 128u;
83
91
 
84
92
  struct LinearAttentionParams {
85
93
  num_tokens: u32,
@@ -102,9 +110,9 @@ struct LinearAttentionParams {
102
110
 
103
111
  @group(0) @binding(0) var<uniform> params: LinearAttentionParams;
104
112
  @group(0) @binding(1) var<storage, read> conv_out: array<f32>;
105
- @group(0) @binding(2) var<storage, read> z_proj: array<f32>;
106
- @group(0) @binding(3) var<storage, read> a_proj: array<f32>;
107
- @group(0) @binding(4) var<storage, read> b_proj: array<f32>;
113
+ @group(0) @binding(2) var<storage, read> z_proj: array<${inputScalar}>;
114
+ @group(0) @binding(3) var<storage, read> a_proj: array<${inputScalar}>;
115
+ @group(0) @binding(4) var<storage, read> b_proj: array<${inputScalar}>;
108
116
  @group(0) @binding(5) var<storage, read> dt_bias: array<f32>;
109
117
  @group(0) @binding(6) var<storage, read> a_neg_exp: array<f32>;
110
118
  @group(0) @binding(7) var<storage, read> norm_weight: array<f32>;
@@ -158,42 +166,52 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
158
166
  let ab_row_base = token_idx * params.num_v_heads + head;
159
167
  let out_row_base = token_idx * params.value_dim + head * head_v_dim;
160
168
 
161
- // L2 norm for Q and K (redundant across threads but avoids shared memory)
162
169
  var q_norm_sq = 0.0;
163
- var k_norm_sq = 0.0;
164
- for (var d: u32 = 0u; d < head_k_dim; d = d + 1u) {
170
+ for (var d: u32 = vd; d < head_k_dim; d = d + WORKGROUP_SIZE) {
165
171
  let q_val = conv_out[conv_row_base + q_base + d];
166
- let k_val = conv_out[conv_row_base + k_base + d];
167
172
  q_norm_sq = q_norm_sq + q_val * q_val;
168
- k_norm_sq = k_norm_sq + k_val * k_val;
169
173
  }
174
+ shared_sq[vd] = q_norm_sq;
175
+ workgroupBarrier();
176
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
177
+ if (vd < stride) {
178
+ shared_sq[vd] = shared_sq[vd] + shared_sq[vd + stride];
179
+ }
180
+ workgroupBarrier();
181
+ }
182
+ let q_norm_scale = head_scale / sqrt(shared_sq[0] + params.qk_l2norm_eps);
170
183
 
171
- let q_norm_scale = head_scale / sqrt(q_norm_sq + params.qk_l2norm_eps);
172
- let k_norm_scale = inverseSqrt(k_norm_sq + params.qk_l2norm_eps);
173
- let beta = 1.0 / (1.0 + exp(-b_proj[ab_row_base]));
174
- let g = a_neg_exp[head] * softplus(a_proj[ab_row_base] + dt_bias[head]);
184
+ var k_norm_sq = 0.0;
185
+ for (var d: u32 = vd; d < head_k_dim; d = d + WORKGROUP_SIZE) {
186
+ let k_val = conv_out[conv_row_base + k_base + d];
187
+ k_norm_sq = k_norm_sq + k_val * k_val;
188
+ }
189
+ shared_sq[vd] = k_norm_sq;
190
+ workgroupBarrier();
191
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
192
+ if (vd < stride) {
193
+ shared_sq[vd] = shared_sq[vd] + shared_sq[vd + stride];
194
+ }
195
+ workgroupBarrier();
196
+ }
197
+ let k_norm_scale = inverseSqrt(shared_sq[0] + params.qk_l2norm_eps);
198
+ let beta = 1.0 / (1.0 + exp(-f32(b_proj[ab_row_base])));
199
+ let g = a_neg_exp[head] * softplus(f32(a_proj[ab_row_base]) + dt_bias[head]);
175
200
  let g_exp = exp(g);
176
201
 
177
- // Gated delta net recurrence (matches HF Qwen3_5 reference):
178
- // S_t = exp(g) * S_{t-1} (decay)
179
- // kv_mem = S_t^T @ k (retrieve from decayed state)
180
- // delta = beta * (v - kv_mem) (error-corrected update)
181
- // S_t = S_t + k @ delta^T (write)
182
- // o_t = S_t^T @ q (read from updated state)
183
202
  if (is_active) {
184
- // Step 1: Decay state
185
203
  for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
186
204
  let state_idx = recurrent_head_base + kd * head_v_dim + vd;
187
205
  recurrent_state[state_idx] = recurrent_state[state_idx] * g_exp;
188
206
  }
189
- // Step 2: Retrieve from decayed state
190
- var kv_mem = 0.0;
207
+ }
208
+ var kv_mem = 0.0;
209
+ if (is_active) {
191
210
  for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
192
211
  let k_normed = conv_out[conv_row_base + k_base + kd] * k_norm_scale;
193
212
  let state_idx = recurrent_head_base + kd * head_v_dim + vd;
194
213
  kv_mem = kv_mem + recurrent_state[state_idx] * k_normed;
195
214
  }
196
- // Step 3-4: Compute delta and write update
197
215
  let delta = (conv_out[conv_row_base + v_base + vd] - kv_mem) * beta;
198
216
  for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
199
217
  let k_normed = conv_out[conv_row_base + k_base + kd] * k_norm_scale;
@@ -202,7 +220,6 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
202
220
  }
203
221
  }
204
222
 
205
- // Output — each thread computes one vd element
206
223
  var out_value = 0.0;
207
224
  if (is_active) {
208
225
  for (var kd: u32 = 0u; kd < head_k_dim; kd = kd + 1u) {
@@ -210,13 +227,13 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
210
227
  let state_idx = recurrent_head_base + kd * head_v_dim + vd;
211
228
  out_value = out_value + recurrent_state[state_idx] * q_normed;
212
229
  }
230
+ }
231
+ if (is_active) {
213
232
  output[out_row_base + vd] = out_value;
214
233
  }
215
234
 
216
- // RMS norm reduction across vd (workgroup-level)
217
235
  shared_sq[vd] = select(0.0, out_value * out_value, is_active);
218
236
  workgroupBarrier();
219
- // Tree reduction
220
237
  for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
221
238
  if (vd < stride) {
222
239
  shared_sq[vd] = shared_sq[vd] + shared_sq[vd + stride];
@@ -225,23 +242,22 @@ fn main(@builtin(workgroup_id) wid: vec3<u32>,
225
242
  }
226
243
  let inv_rms = inverseSqrt(shared_sq[0] / f32(head_v_dim) + params.rms_norm_eps);
227
244
 
228
- // Apply norm + gate
229
245
  if (is_active) {
230
- let gate = silu(z_proj[z_row_base + vd]);
246
+ let gate = silu(f32(z_proj[z_row_base + vd]));
231
247
  let norm_index = select(vd, head * head_v_dim + vd, params.norm_mode == 1u);
232
248
  output[out_row_base + vd] = (output[out_row_base + vd] * inv_rms) * norm_weight[norm_index] * gate;
233
249
  }
234
250
  }
235
251
  }
236
252
  `;
253
+ }
237
254
 
238
255
  let cachedEpoch = -1;
239
- let convPipeline = null;
240
- let recurrentPipeline = null;
256
+ const pipelineCache = new Map();
241
257
  let convBindGroupLayout = null;
242
258
  let recurrentBindGroupLayout = null;
243
259
 
244
- function createPipelines(device) {
260
+ function createBindGroupLayouts(device) {
245
261
  convBindGroupLayout = getOrCreateBindGroupLayout(
246
262
  'linear_attention_conv_layout',
247
263
  [
@@ -269,18 +285,22 @@ function createPipelines(device) {
269
285
  ],
270
286
  device
271
287
  );
288
+ }
272
289
 
290
+ function createPipelines(device, inputDtype) {
291
+ createBindGroupLayouts(device);
292
+ const variant = inputDtype === 'f16' ? 'f16' : 'f32';
273
293
  const convModule = device.createShaderModule({
274
- label: 'linear_attention_conv',
275
- code: CONV_SHADER,
294
+ label: `linear_attention_conv_${variant}`,
295
+ code: buildLinearAttentionConvShader(variant),
276
296
  });
277
297
  const recurrentModule = device.createShaderModule({
278
- label: 'linear_attention_recurrent',
279
- code: RECURRENT_SHADER,
298
+ label: `linear_attention_recurrent_${variant}`,
299
+ code: buildLinearAttentionRecurrentShader(variant),
280
300
  });
281
301
 
282
- convPipeline = device.createComputePipeline({
283
- label: 'linear_attention_conv_pipeline',
302
+ const convPipeline = device.createComputePipeline({
303
+ label: `linear_attention_conv_pipeline_${variant}`,
284
304
  layout: getOrCreatePipelineLayout('linear_attention_conv_pipeline_layout', [convBindGroupLayout], device),
285
305
  compute: {
286
306
  module: convModule,
@@ -290,8 +310,8 @@ function createPipelines(device) {
290
310
  },
291
311
  },
292
312
  });
293
- recurrentPipeline = device.createComputePipeline({
294
- label: 'linear_attention_recurrent_pipeline',
313
+ const recurrentPipeline = device.createComputePipeline({
314
+ label: `linear_attention_recurrent_pipeline_${variant}`,
295
315
  layout: getOrCreatePipelineLayout('linear_attention_recurrent_pipeline_layout', [recurrentBindGroupLayout], device),
296
316
  compute: {
297
317
  module: recurrentModule,
@@ -301,14 +321,28 @@ function createPipelines(device) {
301
321
  },
302
322
  },
303
323
  });
324
+
325
+ pipelineCache.set(variant, { convPipeline, recurrentPipeline });
326
+ }
327
+
328
+ function normalizeInputDtype(dtype) {
329
+ return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype });
304
330
  }
305
331
 
306
- function ensurePipelines(device) {
332
+ function ensurePipelines(device, inputDtype) {
307
333
  const epoch = getDeviceEpoch();
308
- if (epoch !== cachedEpoch || !convPipeline || !recurrentPipeline) {
309
- createPipelines(device);
334
+ if (epoch !== cachedEpoch) {
335
+ pipelineCache.clear();
336
+ convBindGroupLayout = null;
337
+ recurrentBindGroupLayout = null;
338
+ cachedEpoch = epoch;
339
+ }
340
+ const variant = normalizeInputDtype(inputDtype);
341
+ if (!pipelineCache.has(variant)) {
342
+ createPipelines(device, variant);
310
343
  cachedEpoch = epoch;
311
344
  }
345
+ return pipelineCache.get(variant) ?? null;
312
346
  }
313
347
 
314
348
  function buildParamsData(params) {
@@ -339,6 +373,19 @@ function requireGpuBuffer(buffer, label) {
339
373
  }
340
374
  }
341
375
 
376
+ function resolveOutputDtype(outputDtype) {
377
+ const normalized = selectRuleValue(
378
+ 'shared',
379
+ 'dtype',
380
+ 'f16OrF32FromDtype',
381
+ { dtype: outputDtype === undefined ? 'f32' : outputDtype }
382
+ );
383
+ if (normalized === 'f16' || normalized === 'f32') {
384
+ return normalized;
385
+ }
386
+ throw new Error(`linear_attention core output dtype "${outputDtype}" is invalid.`);
387
+ }
388
+
342
389
  export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTensor, layerState, options = {}) {
343
390
  const device = getDevice();
344
391
  if (!device) {
@@ -371,12 +418,28 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
371
418
  throw new Error(`linear_attention requires supported normMode, got ${layerState.normMode}.`);
372
419
  }
373
420
 
374
- ensurePipelines(device);
421
+ const inputDtype = normalizeInputDtype(qkvTensor?.dtype);
422
+ if (normalizeInputDtype(zTensor?.dtype) !== inputDtype) {
423
+ throw new Error(`linear_attention core requires matching qkv/z dtypes; got ${qkvTensor?.dtype} and ${zTensor?.dtype}.`);
424
+ }
425
+ if (normalizeInputDtype(aTensor?.dtype) !== inputDtype) {
426
+ throw new Error(`linear_attention core requires matching qkv/a dtypes; got ${qkvTensor?.dtype} and ${aTensor?.dtype}.`);
427
+ }
428
+ if (normalizeInputDtype(bTensor?.dtype) !== inputDtype) {
429
+ throw new Error(`linear_attention core requires matching qkv/b dtypes; got ${qkvTensor?.dtype} and ${bTensor?.dtype}.`);
430
+ }
431
+
432
+ const pipelines = ensurePipelines(device, inputDtype);
433
+ if (!pipelines) {
434
+ throw new Error(`linear_attention core failed to resolve pipelines for dtype "${inputDtype}".`);
435
+ }
375
436
 
376
437
  const convOutSize = numTokens * layerState.convDim * Float32Array.BYTES_PER_ELEMENT;
377
438
  const outputSize = numTokens * layerState.valueDim * Float32Array.BYTES_PER_ELEMENT;
378
439
  const convOutBuffer = acquireBuffer(convOutSize, undefined, `L${options.layerIdx ?? 0}.linear_conv_out`);
379
440
  const outputBuffer = acquireBuffer(outputSize, undefined, `L${options.layerIdx ?? 0}.linear_attention_core_out`);
441
+ const outputDtype = resolveOutputDtype(options.outputDtype);
442
+ const outputShape = [numTokens, layerState.valueDim];
380
443
  if (useRecorder) {
381
444
  const paramsBuffer = createUniformBufferFromData(
382
445
  'linear_attention_params',
@@ -430,14 +493,14 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
430
493
 
431
494
  recordDispatch(
432
495
  recorder,
433
- convPipeline,
496
+ pipelines.convPipeline,
434
497
  convBindGroup,
435
498
  [Math.ceil(layerState.convDim / CONV_WORKGROUP_SIZE), 1, 1],
436
499
  'linear_attention_conv'
437
500
  );
438
501
  recordDispatch(
439
502
  recorder,
440
- recurrentPipeline,
503
+ pipelines.recurrentPipeline,
441
504
  recurrentBindGroup,
442
505
  [layerState.numVHeads, 1, 1],
443
506
  'linear_attention_recurrent'
@@ -445,12 +508,25 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
445
508
 
446
509
  recorder.trackTemporaryBuffer(convOutBuffer);
447
510
 
448
- return createTensor(
511
+ const output = createTensor(
449
512
  outputBuffer,
450
513
  'f32',
451
- [numTokens, layerState.valueDim],
514
+ outputShape,
452
515
  `L${options.layerIdx ?? 0}.linear_attention_core`
453
516
  );
517
+ if (outputDtype === 'f16') {
518
+ assertImplicitDtypeTransitionAllowed({
519
+ executionPolicies: options.executionPolicies ?? null,
520
+ fromDtype: output.dtype,
521
+ toDtype: 'f16',
522
+ op: 'linear_attention_core',
523
+ detail: 'Linear attention core would narrow activations implicitly.',
524
+ });
525
+ const casted = await recordCastF32ToF16(recorder, output);
526
+ recorder.trackTemporaryBuffer(outputBuffer);
527
+ return casted;
528
+ }
529
+ return output;
454
530
  } catch (error) {
455
531
  releaseBuffer(convOutBuffer);
456
532
  releaseBuffer(outputBuffer);
@@ -516,7 +592,7 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
516
592
 
517
593
  {
518
594
  const pass = encoder.beginComputePass({ label: 'linear_attention_conv_pass' });
519
- pass.setPipeline(convPipeline);
595
+ pass.setPipeline(pipelines.convPipeline);
520
596
  pass.setBindGroup(0, convBindGroup);
521
597
  pass.dispatchWorkgroups(Math.ceil(layerState.convDim / CONV_WORKGROUP_SIZE), 1, 1);
522
598
  pass.end();
@@ -524,7 +600,7 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
524
600
 
525
601
  {
526
602
  const pass = encoder.beginComputePass({ label: 'linear_attention_recurrent_pass' });
527
- pass.setPipeline(recurrentPipeline);
603
+ pass.setPipeline(pipelines.recurrentPipeline);
528
604
  pass.setBindGroup(0, recurrentBindGroup);
529
605
  pass.dispatchWorkgroups(layerState.numVHeads, 1, 1);
530
606
  pass.end();
@@ -533,12 +609,25 @@ export async function runLinearAttentionCoreGPU(qkvTensor, zTensor, aTensor, bTe
533
609
  device.queue.submit([encoder.finish()]);
534
610
  submitted = true;
535
611
 
536
- return createTensor(
612
+ const output = createTensor(
537
613
  outputBuffer,
538
614
  'f32',
539
- [numTokens, layerState.valueDim],
615
+ outputShape,
540
616
  `L${options.layerIdx ?? 0}.linear_attention_core`
541
617
  );
618
+ if (outputDtype === 'f16') {
619
+ assertImplicitDtypeTransitionAllowed({
620
+ executionPolicies: options.executionPolicies ?? null,
621
+ fromDtype: output.dtype,
622
+ toDtype: 'f16',
623
+ op: 'linear_attention_core',
624
+ detail: 'Linear attention core would narrow activations implicitly.',
625
+ });
626
+ const casted = await castF32ToF16(output);
627
+ releaseBuffer(outputBuffer);
628
+ return casted;
629
+ }
630
+ return output;
542
631
  } catch (error) {
543
632
  releaseBuffer(outputBuffer);
544
633
  throw error;
@@ -61,6 +61,9 @@ export function calculateMatmulDispatch(variant, useQ4KFused, useGemv, M, N, con
61
61
  if (variant === 'q4_fused') {
62
62
  workgroupsX = N;
63
63
  workgroupsY = 1;
64
+ } else if (config.variantMetadata?.colsPerWg && config.variantMetadata?.tileM) {
65
+ workgroupsX = Math.ceil(N / colsPerWg);
66
+ workgroupsY = Math.ceil(M / tileM);
64
67
  } else if (config.variantMetadata?.colsPerWg) {
65
68
  // Multicol variants: q4_fused_multicol, q4_fused_multicol_f16
66
69
  workgroupsX = Math.ceil(N / colsPerWg);
@@ -7,11 +7,9 @@ import { ALIGNMENT, QUANTIZATION, TILE_SIZES } from './constants.js';
7
7
  import { getKernelConfig, hasRequiredFeatures } from './utils.js';
8
8
  import { getKernelThresholds } from '../../config/schema/index.js';
9
9
  import {
10
- getActiveKernelPath,
11
10
  getKernelPathMatmulConstants,
12
11
  getKernelPathMatmulVariant,
13
12
  getKernelPathStrict,
14
- isKernelPathFusedQ4K,
15
13
  } from '../../config/kernel-path-loader.js';
16
14
  import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
17
15
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
@@ -22,10 +20,11 @@ import { logKernelSelectionOnce } from '../kernel-selection-log.js';
22
20
  // =============================================================================
23
21
 
24
22
 
25
- function selectQ4KFusedVariant(isM1, wantF16Output, aDtype) {
23
+ function selectQ4KFusedVariant(isM1, wantF16Output, aDtype, phase) {
26
24
  const useF16A = wantF16Output && aDtype === 'f16';
27
25
  const useF16Out = wantF16Output && aDtype !== 'f16';
28
- return selectKernelRuleValue('matmul', 'q4kFusedVariant', { useF16A, useF16Out, isM1 });
26
+ const isPrefill = phase === 'prefill' && !isM1;
27
+ return selectKernelRuleValue('matmul', 'q4kFusedVariant', { useF16A, useF16Out, isM1, isPrefill });
29
28
  }
30
29
 
31
30
 
@@ -95,19 +94,10 @@ export function isFusedQ4KDisabled(options = {}) {
95
94
  if (options.disableFusedQ4K === true) return true;
96
95
  const capabilities = getKernelCapabilities();
97
96
  const hasSubgroups = capabilities?.hasSubgroups === true;
98
-
99
- if (options.kernelPath !== undefined) {
100
- if (!options.kernelPath) {
101
- return !hasSubgroups;
102
- }
103
- return !isKernelPathFusedQ4K(options.kernelPath);
104
- }
105
-
106
- const activeKernelPath = getActiveKernelPath();
107
- if (activeKernelPath) {
108
- return !isKernelPathFusedQ4K(activeKernelPath);
109
- }
110
-
97
+ // Subgroups are the only hardware gate for fused Q4K. When available,
98
+ // fused Q4K is always selectable regardless of kernel path contents.
99
+ // The preferredWeightDtype rule decides whether to route weights to Q4K;
100
+ // this function only gates whether the fused variant is *allowed*.
111
101
  return !hasSubgroups;
112
102
  }
113
103
 
@@ -254,27 +244,7 @@ export function requiresF32Input(variant) {
254
244
  }
255
245
 
256
246
  function resolveRequiredWeightDtype(config) {
257
- const shaderFile = String(config?.shaderFile ?? config?.wgsl ?? '');
258
- if (!shaderFile) {
259
- return null;
260
- }
261
- if (shaderFile.startsWith('fused_matmul_q4')) {
262
- return 'q4k';
263
- }
264
- if (
265
- shaderFile === 'matmul_f16.wgsl'
266
- || shaderFile === 'matmul_f16_tiled.wgsl'
267
- || shaderFile === 'matmul_f16w_f32a.wgsl'
268
- || shaderFile === 'matmul_f16w_f32a_tiled.wgsl'
269
- || shaderFile === 'matmul_gemv_subgroup.wgsl'
270
- || shaderFile === 'matmul_gemv_subgroup_f16a.wgsl'
271
- ) {
272
- return 'f16';
273
- }
274
- if (shaderFile === 'matmul_f32.wgsl') {
275
- return 'f32';
276
- }
277
- return null;
247
+ return config?.weightDtype ?? null;
278
248
  }
279
249
 
280
250
 
@@ -316,15 +286,21 @@ function resolveMatmulOverride(
316
286
  }
317
287
  if (requestedOutputDtype && outputDtype !== requestedOutputDtype) {
318
288
  return failOrWarn(
319
- `Matmul kernel "${variantOverride}" outputs ${outputDtype} but ${requestedOutputDtype} was requested.`
289
+ `Matmul kernel "${variantOverride}" outputs ${outputDtype} but ${requestedOutputDtype} was requested. [M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype}]`
320
290
  );
321
291
  }
322
292
 
323
293
  const requiredWeightDtype = resolveRequiredWeightDtype(config);
324
294
  const weightDtypeOk = !requiredWeightDtype
325
- || bDtype === requiredWeightDtype
326
- || (requiredWeightDtype === 'f16' && bDtype === 'q4k');
295
+ || bDtype === requiredWeightDtype;
327
296
  if (!weightDtypeOk) {
297
+ const overridePolicy = selectKernelRuleValue(
298
+ 'matmul', 'weightDtypeMismatchPolicy',
299
+ { requiredWeightDtype, actualWeightDtype: bDtype }
300
+ );
301
+ if (overridePolicy === 'fallthrough') {
302
+ return null;
303
+ }
328
304
  return failOrWarn(
329
305
  `Matmul kernel "${variantOverride}" requires ${requiredWeightDtype} weights but B dtype is ${bDtype}.`
330
306
  );
@@ -407,6 +383,19 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
407
383
  strict,
408
384
  fusedQ4KDisabled
409
385
  );
386
+ if (!override && strict) {
387
+ // When weights resolved to Q4K (precision upgrade) but the path variant requires F16,
388
+ // fall through to auto-selection rather than throwing. The auto path will pick
389
+ // the correct fused Q4K variant for the actual weight dtype.
390
+ if (bDtype === 'q4k') {
391
+ logKernelSelectionOnce('matmul', {
392
+ variant: pathVariant,
393
+ reason: 'path_override_q4k_fallthrough',
394
+ });
395
+ } else {
396
+ throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
397
+ }
398
+ }
410
399
  if (override) {
411
400
  if (
412
401
  phase === 'prefill'
@@ -453,7 +442,7 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
453
442
  const isQ4K = bDtype === 'q4k';
454
443
  const wantF16Output = requestedOutputDtype === 'f16' && capabilities.hasF16;
455
444
  const q4kVariant = isQ4K && capabilities.hasSubgroups && fusedAllowed
456
- ? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype)
445
+ ? selectQ4KFusedVariant(M === 1, wantF16Output, aDtype, phase)
457
446
  : null;
458
447
 
459
448
  const effectiveBDtype = bDtype === 'q4k' ? 'f32' : bDtype;
@@ -14,6 +14,7 @@ import type { CommandRecorder } from '../command-recorder.js';
14
14
  import type { OutputBufferOptions, OutputDtypeOptions, Vec4Options } from './types.js';
15
15
  import type { KernelPathSchema } from '../../config/schema/index.js';
16
16
  import type { MatmulDebugConfigSchema } from '../../config/schema/debug.schema.js';
17
+ import type { ExecutionV1PoliciesSchema } from '../../config/schema/execution-v1.schema.js';
17
18
 
18
19
  /** Matmul kernel options */
19
20
  export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions, Vec4Options {
@@ -43,6 +44,8 @@ export interface MatmulOptions extends OutputBufferOptions, OutputDtypeOptions,
43
44
  constants?: Record<string, number | boolean>;
44
45
  /** Runtime debug controls for attention projection diagnostics. */
45
46
  matmulDebug?: MatmulDebugConfigSchema | null;
47
+ /** Execution-v1 fail-fast policies for implicit dtype transitions. */
48
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
46
49
  }
47
50
 
48
51
  /** Context for base matmul kernel selection rules. */