doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -31,6 +31,21 @@ const SUBGROUP_REQUIRING_FILES = new Set([
31
31
  'attention_decode_online_f16.wgsl',
32
32
  ]);
33
33
 
34
+ const KERNEL_FILE_PRECISION_PATCHES = new Map([
35
+ ['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
36
+ ['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
37
+ ['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
38
+ ['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
39
+ ['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
40
+ ['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
41
+ ['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
42
+ ['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
43
+ ['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
44
+ ['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
45
+ ['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
46
+ ['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
47
+ ]);
48
+
34
49
  /**
35
50
  * Check whether a kernel entry requires subgroup support.
36
51
  * @param {{ kernel: string }} kernelEntry
@@ -41,6 +56,11 @@ function isSubgroupKernel(kernelEntry) {
41
56
  return kernelEntry.kernel.includes('subgroup') || SUBGROUP_REQUIRING_FILES.has(kernelEntry.kernel);
42
57
  }
43
58
 
59
+ function requiresNoSubgroupFallback(kernelEntry) {
60
+ if (typeof kernelEntry?.kernel !== 'string') return false;
61
+ return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
62
+ }
63
+
44
64
  /**
45
65
  * Find all kernel keys in the graph whose `kernel` file matches the given filename.
46
66
  * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
@@ -82,9 +102,39 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
82
102
  } else if (constants !== undefined) {
83
103
  derived.constants = { ...constants };
84
104
  }
105
+ const precision = deriveKernelPrecision(base, newFile);
106
+ if (precision) {
107
+ derived.precision = precision;
108
+ } else {
109
+ delete derived.precision;
110
+ }
85
111
  return derived;
86
112
  }
87
113
 
114
+ function deriveKernelPrecision(base, newFile) {
115
+ const precision = base.precision ? { ...base.precision } : {};
116
+ const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
117
+ if (precisionPatch) {
118
+ Object.assign(precision, precisionPatch);
119
+ }
120
+ if (!String(newFile).startsWith('attention')) {
121
+ return Object.keys(precision).length > 0 ? precision : null;
122
+ }
123
+ if (newFile.includes('_f16kv')) {
124
+ precision.activationDtype = 'f32';
125
+ precision.kvDtype = 'f16';
126
+ return precision;
127
+ }
128
+ if (newFile.includes('_f16')) {
129
+ precision.activationDtype = 'f16';
130
+ precision.kvDtype = 'f16';
131
+ return precision;
132
+ }
133
+ precision.activationDtype = 'f32';
134
+ precision.kvDtype = 'f32';
135
+ return precision;
136
+ }
137
+
88
138
  /**
89
139
  * Derive a non-colliding kernel key name.
90
140
  * @param {object} kernels - existing kernels dict
@@ -136,6 +186,292 @@ function stepUsesFile(graph, step, filename) {
136
186
  return entry != null && entry.kernel === filename;
137
187
  }
138
188
 
189
+ /**
190
+ * Find the first kernel key used by matching ops in a phase whose shader file
191
+ * satisfies the provided predicate.
192
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
193
+ * @param {Array<Array>} steps
194
+ * @param {ReadonlySet<string>} ops
195
+ * @param {(entry: { kernel: string, entry: string }) => boolean} predicate
196
+ * @returns {string | null}
197
+ */
198
+ function findPhaseKernelKey(graph, steps, ops, predicate) {
199
+ for (const step of steps || []) {
200
+ if (!ops.has(step[0])) {
201
+ continue;
202
+ }
203
+ const entry = graph.kernels[step[1]];
204
+ if (entry && predicate(entry)) {
205
+ return step[1];
206
+ }
207
+ }
208
+ return null;
209
+ }
210
+
211
+ /**
212
+ * Find an existing kernel key by shader file and entry point.
213
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
214
+ * @param {string} filename
215
+ * @param {string} entryPoint
216
+ * @returns {string | null}
217
+ */
218
+ function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
219
+ for (const [key, entry] of Object.entries(graph.kernels)) {
220
+ if (entry.kernel === filename && entry.entry === entryPoint) {
221
+ return key;
222
+ }
223
+ }
224
+ return null;
225
+ }
226
+
227
+ function normalizeLayerType(layerType) {
228
+ return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
229
+ }
230
+
231
+ function isLinearAttentionLayerType(layerType) {
232
+ const normalized = normalizeLayerType(layerType);
233
+ return normalized === 'linear_attention'
234
+ || normalized === 'linear'
235
+ || normalized === 'gated_delta'
236
+ || normalized === 'gated_delta_net';
237
+ }
238
+
239
+ function isFullAttentionLayerType(layerType) {
240
+ const normalized = normalizeLayerType(layerType);
241
+ return normalized === 'full_attention'
242
+ || normalized === 'full'
243
+ || normalized === 'global'
244
+ || normalized === 'standard';
245
+ }
246
+
247
+ function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey) {
248
+ const groupedEntries = [];
249
+ if (!Array.isArray(baseStep) || baseStep.length < 2) {
250
+ return groupedEntries;
251
+ }
252
+
253
+ const totalLayers = targetLayers.allLayers;
254
+ const targeted = targetLayers.matchingLayers;
255
+ const remaining = totalLayers.filter((layerIdx) => !targeted.includes(layerIdx));
256
+
257
+ if (remaining.length > 0) {
258
+ groupedEntries.push({
259
+ layers: remaining,
260
+ steps: [baseStep],
261
+ });
262
+ }
263
+ if (targeted.length > 0) {
264
+ const replacement = [...baseStep];
265
+ replacement[1] = replacementKernelKey;
266
+ groupedEntries.push({
267
+ layers: targeted,
268
+ steps: [replacement],
269
+ });
270
+ }
271
+
272
+ return groupedEntries;
273
+ }
274
+
275
+ function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
276
+ if (!Array.isArray(steps) || steps.length === 0 || !replacementKernelKey) {
277
+ return { steps, changed: false };
278
+ }
279
+ let changed = false;
280
+ const nextSteps = steps.map((step) => {
281
+ if (!Array.isArray(step) || step[0] !== op) {
282
+ return step;
283
+ }
284
+ if (step[1] === replacementKernelKey) {
285
+ return step;
286
+ }
287
+ const replacement = [...step];
288
+ replacement[1] = replacementKernelKey;
289
+ changed = true;
290
+ return replacement;
291
+ });
292
+ return { steps: nextSteps, changed };
293
+ }
294
+
295
+ function deriveKernelEntryWithPrecision(base, precision) {
296
+ return {
297
+ ...base,
298
+ precision: {
299
+ ...(base.precision ?? {}),
300
+ ...precision,
301
+ },
302
+ };
303
+ }
304
+
305
+ const ATTENTION_F16KV_TO_F16_MAP = new Map([
306
+ ['attention_decode_online_f16kv.wgsl', 'attention_decode_online_f16.wgsl'],
307
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
308
+ ['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
309
+ ['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
310
+ ]);
311
+
312
+ function deriveF16AttentionKernelEntry(base) {
313
+ if (typeof base?.kernel !== 'string') {
314
+ return null;
315
+ }
316
+ const replacement = ATTENTION_F16KV_TO_F16_MAP.get(base.kernel);
317
+ if (!replacement) {
318
+ return null;
319
+ }
320
+ return deriveKernelEntryWithPrecision(
321
+ deriveKernelEntry(base, replacement, base.entry),
322
+ { activationDtype: 'f16', kvDtype: 'f16' }
323
+ );
324
+ }
325
+
326
+ function deriveLinearDecodeF16KernelEntry(base) {
327
+ const precision = {
328
+ inputDtype: 'f16',
329
+ outputDtype: 'f16',
330
+ };
331
+ if (base.kernel === 'fused_matmul_q4.wgsl' && base.entry === 'main_multicol') {
332
+ return {
333
+ ...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a'),
334
+ precision: {
335
+ ...(base.precision ?? {}),
336
+ ...precision,
337
+ },
338
+ };
339
+ }
340
+ if (
341
+ (base.kernel === 'fused_matmul_q4_multicol_f16.wgsl' && base.entry === 'main_multicol_f16')
342
+ || (base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl' && base.entry === 'main_multicol_f16a')
343
+ ) {
344
+ return deriveKernelEntryWithPrecision(base, precision);
345
+ }
346
+ return null;
347
+ }
348
+
349
+ function deriveLmHeadDecodeF16KernelEntry(base) {
350
+ const precision = {
351
+ inputDtype: 'f16',
352
+ outputDtype: 'f16',
353
+ };
354
+ if (base.kernel === 'matmul_gemv_subgroup.wgsl' && base.entry === 'main_multicol') {
355
+ return {
356
+ ...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', 'main_multicol'),
357
+ precision: {
358
+ ...(base.precision ?? {}),
359
+ ...precision,
360
+ },
361
+ };
362
+ }
363
+ if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl' && base.entry === 'main_multicol') {
364
+ return deriveKernelEntryWithPrecision(base, precision);
365
+ }
366
+ return null;
367
+ }
368
+
369
+ function deriveQ4DecodeF16KernelEntry(base) {
370
+ if (typeof base?.kernel !== 'string') {
371
+ return null;
372
+ }
373
+ const precision = {
374
+ inputDtype: 'f16',
375
+ outputDtype: 'f16',
376
+ };
377
+ if (base.kernel === 'fused_matmul_q4.wgsl') {
378
+ return {
379
+ ...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a', null),
380
+ precision: {
381
+ ...(base.precision ?? {}),
382
+ ...precision,
383
+ },
384
+ };
385
+ }
386
+ if (
387
+ base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
388
+ || base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
389
+ ) {
390
+ return deriveKernelEntryWithPrecision(base, precision);
391
+ }
392
+ return null;
393
+ }
394
+
395
+ function deriveQ4PrefillF16KernelEntry(base) {
396
+ if (typeof base?.kernel !== 'string') {
397
+ return null;
398
+ }
399
+ const precision = {
400
+ inputDtype: 'f16',
401
+ outputDtype: 'f16',
402
+ };
403
+ if (base.kernel.startsWith('fused_matmul_q4_batched')) {
404
+ return {
405
+ ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
406
+ precision: {
407
+ ...(base.precision ?? {}),
408
+ ...precision,
409
+ },
410
+ };
411
+ }
412
+ return null;
413
+ }
414
+
415
+ function deriveQ4DecodeF32ActivationKernelEntry(base) {
416
+ if (typeof base?.kernel !== 'string') {
417
+ return null;
418
+ }
419
+ const precision = {
420
+ inputDtype: 'f32',
421
+ outputDtype: 'f32',
422
+ };
423
+ if (
424
+ base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
425
+ || base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
426
+ ) {
427
+ return deriveKernelEntryWithPrecision(
428
+ deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main_multicol', null),
429
+ precision
430
+ );
431
+ }
432
+ if (base.kernel === 'fused_matmul_q4_f16a.wgsl') {
433
+ return deriveKernelEntryWithPrecision(
434
+ deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main', null),
435
+ precision
436
+ );
437
+ }
438
+ return null;
439
+ }
440
+
441
+ function deriveQ4PrefillF32ActivationKernelEntry(base) {
442
+ if (typeof base?.kernel !== 'string') {
443
+ return null;
444
+ }
445
+ if (
446
+ base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
447
+ && base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
448
+ ) {
449
+ return null;
450
+ }
451
+ return deriveKernelEntryWithPrecision(
452
+ deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
453
+ { inputDtype: 'f32', outputDtype: 'f32' }
454
+ );
455
+ }
456
+
457
+ function replacePhaseStepEntries(steps, op, replacementEntries) {
458
+ if (!Array.isArray(steps) || steps.length === 0 || !Array.isArray(replacementEntries) || replacementEntries.length === 0) {
459
+ return { steps, changed: false };
460
+ }
461
+ const stepIndex = steps.findIndex((entry) => Array.isArray(entry) && entry[0] === op);
462
+ if (stepIndex === -1) {
463
+ return { steps, changed: false };
464
+ }
465
+ return {
466
+ steps: [
467
+ ...steps.slice(0, stepIndex),
468
+ ...replacementEntries,
469
+ ...steps.slice(stepIndex + 1),
470
+ ],
471
+ changed: true,
472
+ };
473
+ }
474
+
139
475
  // =============================================================================
140
476
  // Transform: removeSubgroups
141
477
  // =============================================================================
@@ -151,8 +487,8 @@ function stepUsesFile(graph, step, filename) {
151
487
  * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
152
488
  */
153
489
  export function removeSubgroups(graph, ctx) {
154
- const hasAnySubgroup = Object.values(graph.kernels).some(isSubgroupKernel);
155
- if (!hasAnySubgroup) {
490
+ const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
491
+ if (!hasAnyFallbackKernel) {
156
492
  return null;
157
493
  }
158
494
 
@@ -160,15 +496,16 @@ export function removeSubgroups(graph, ctx) {
160
496
  const keyMap = new Map();
161
497
  const isF16Activation = ctx.activationDtype === 'f16';
162
498
 
163
- // Build replacement kernel entries for each subgroup kernel reference
164
- // found in decode and postLayer steps.
499
+ // Build replacement kernel entries for each subgroup or fused-Q4K kernel
500
+ // reference found in decode, prefill, and postLayer steps.
165
501
  const decodeKeys = new Set((result.decode || []).map((s) => s[1]));
502
+ const prefillKeys = new Set((result.prefill || []).map((s) => s[1]));
166
503
  const postLayerKeys = new Set((result.postLayer || []).map((s) => s[1]));
167
- const relevantKeys = new Set([...decodeKeys, ...postLayerKeys]);
504
+ const relevantKeys = new Set([...decodeKeys, ...prefillKeys, ...postLayerKeys]);
168
505
 
169
506
  for (const key of relevantKeys) {
170
507
  const entry = result.kernels[key];
171
- if (!entry || !isSubgroupKernel(entry)) {
508
+ if (!entry || !requiresNoSubgroupFallback(entry)) {
172
509
  continue;
173
510
  }
174
511
 
@@ -205,6 +542,9 @@ export function removeSubgroups(graph, ctx) {
205
542
  } else if (entry.kernel === 'attention_decode_online_f16.wgsl') {
206
543
  newFile = 'attention_decode_chunked_f16.wgsl';
207
544
  newEntry = entry.entry;
545
+ } else if (entry.kernel.startsWith('fused_matmul_q4')) {
546
+ newFile = isF16Activation ? 'matmul_f16_tiled.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
547
+ newConstants = null;
208
548
  } else {
209
549
  // Unknown subgroup kernel — skip
210
550
  continue;
@@ -219,8 +559,9 @@ export function removeSubgroups(graph, ctx) {
219
559
  return null;
220
560
  }
221
561
 
222
- // Remap decode and postLayer steps; leave prefill and preLayer untouched
562
+ // Remap decode, prefill, and postLayer steps; leave preLayer untouched
223
563
  result.decode = remapStepKeys(result.decode || [], keyMap);
564
+ result.prefill = remapStepKeys(result.prefill || [], keyMap);
224
565
  result.postLayer = remapStepKeys(result.postLayer || [], keyMap);
225
566
 
226
567
  return result;
@@ -252,6 +593,55 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
252
593
  ['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
253
594
  ]);
254
595
 
596
+ /**
597
+ * Activation-only narrowing: f32-activation shaders that still consume f16
598
+ * weights/KV are rewritten onto the matching f16-activation lane.
599
+ *
600
+ * This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
601
+ * runtime session explicitly requests f16 activations for an execution-v1
602
+ * graph that was authored with conservative f32 activation defaults.
603
+ * @type {ReadonlyMap<string, string>}
604
+ */
605
+ const F32_TO_F16_ACTIVATION_MAP = new Map(
606
+ Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
607
+ );
608
+
609
+ function hasExplicitF32ActivationContract(entry) {
610
+ const precision = entry?.precision;
611
+ if (!precision || typeof precision !== 'object') {
612
+ return false;
613
+ }
614
+ return precision.activationDtype === 'f32'
615
+ || precision.inputDtype === 'f32'
616
+ || precision.outputDtype === 'f32';
617
+ }
618
+
619
+ /**
620
+ * Correctness fallback: preserve f16 weights where possible, but widen both
621
+ * activations and KV-cache interactions onto the stable f32 execution lane.
622
+ * Used for alternate-plan recovery after finiteness failure.
623
+ * @type {ReadonlyMap<string, string>}
624
+ */
625
+ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
626
+ ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
627
+ ['rope_f16.wgsl', 'rope.wgsl'],
628
+ ['residual_f16.wgsl', 'residual.wgsl'],
629
+ ['gelu_f16.wgsl', 'gelu.wgsl'],
630
+ ['sample_f16.wgsl', 'sample.wgsl'],
631
+ ['gather_f16.wgsl', 'gather.wgsl'],
632
+ ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
633
+ ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
634
+ ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
635
+ ['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
636
+ ['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
637
+ ['attention_small_f16.wgsl', 'attention_small.wgsl'],
638
+ ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
639
+ ['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
640
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
641
+ ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
642
+ ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
643
+ ]);
644
+
255
645
  /**
256
646
  * Full f32 widening: every shader that uses `enable f16;` is replaced with a
257
647
  * pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
@@ -284,6 +674,7 @@ const FULL_F32_SHADER_MAP = new Map([
284
674
  ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
285
675
  ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
286
676
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
677
+ ['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
287
678
  ]);
288
679
 
289
680
  /**
@@ -292,7 +683,7 @@ const FULL_F32_SHADER_MAP = new Map([
292
683
  * Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
293
684
  * equivalent exists) or if no f16 activation shaders are present.
294
685
  *
295
- * NOTE: The caller is responsible for also updating sessionDefaults.activationDtype
686
+ * NOTE: The caller is responsible for also updating session.activationDtype
296
687
  * to reflect the widened dtype.
297
688
  *
298
689
  * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
@@ -322,6 +713,12 @@ export function widenToF32Activations(graph, ctx) {
322
713
  const result = cloneGraph(graph);
323
714
 
324
715
  for (const [key, entry] of Object.entries(result.kernels)) {
716
+ const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
717
+ ?? deriveQ4PrefillF32ActivationKernelEntry(entry);
718
+ if (q4FallbackEntry) {
719
+ result.kernels[key] = q4FallbackEntry;
720
+ continue;
721
+ }
325
722
  const replacement = shaderMap.get(entry.kernel);
326
723
  if (replacement !== undefined) {
327
724
  result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
@@ -331,6 +728,78 @@ export function widenToF32Activations(graph, ctx) {
331
728
  return result;
332
729
  }
333
730
 
731
+ /**
732
+ * Widen an f16 execution graph onto the stable f32 correctness lane used for
733
+ * alternate-plan recovery after finiteness failure.
734
+ *
735
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
736
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
737
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
738
+ */
739
+ export function widenToF32CorrectnessFallback(graph, ctx) {
740
+ if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
741
+ return null;
742
+ }
743
+
744
+ const hasTargetShader = Object.values(graph.kernels).some(
745
+ (entry) => F16_TO_F32_CORRECTNESS_FALLBACK_MAP.has(entry.kernel)
746
+ );
747
+ if (!hasTargetShader) {
748
+ return null;
749
+ }
750
+
751
+ const result = cloneGraph(graph);
752
+ for (const [key, entry] of Object.entries(result.kernels)) {
753
+ const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
754
+ ?? deriveQ4PrefillF32ActivationKernelEntry(entry);
755
+ if (q4FallbackEntry) {
756
+ result.kernels[key] = q4FallbackEntry;
757
+ continue;
758
+ }
759
+ const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
760
+ if (replacement !== undefined) {
761
+ result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
762
+ }
763
+ }
764
+ return result;
765
+ }
766
+
767
+ /**
768
+ * Narrow f32-activation shaders back onto their f16-activation equivalents.
769
+ *
770
+ * Returns null if the graph has no supported f32-activation kernels to swap or
771
+ * if the runtime did not explicitly request f16 activations on an f16-capable
772
+ * GPU.
773
+ *
774
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
775
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
776
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
777
+ */
778
+ export function narrowToF16Activations(graph, ctx) {
779
+ if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
780
+ return null;
781
+ }
782
+
783
+ const hasTargetShader = Object.values(graph.kernels).some(
784
+ (entry) => !hasExplicitF32ActivationContract(entry) && F32_TO_F16_ACTIVATION_MAP.has(entry.kernel)
785
+ );
786
+ if (!hasTargetShader) {
787
+ return null;
788
+ }
789
+
790
+ const result = cloneGraph(graph);
791
+ for (const [key, entry] of Object.entries(result.kernels)) {
792
+ if (hasExplicitF32ActivationContract(entry)) {
793
+ continue;
794
+ }
795
+ const replacement = F32_TO_F16_ACTIVATION_MAP.get(entry.kernel);
796
+ if (replacement !== undefined) {
797
+ result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
798
+ }
799
+ }
800
+ return result;
801
+ }
802
+
334
803
  // =============================================================================
335
804
  // Transform: swapPrefillAttention
336
805
  // =============================================================================
@@ -390,6 +859,30 @@ export function swapPrefillAttention(graph, ctx, opts) {
390
859
  return changed ? result : null;
391
860
  }
392
861
 
862
+ // =============================================================================
863
+ // Transform: useHead256PrefillAttention
864
+ // =============================================================================
865
+
866
+ /**
867
+ * Promote prefill attention onto the fixed 256-dim shared-block kernel.
868
+ *
869
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
870
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
871
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
872
+ */
873
+ export function useHead256PrefillAttention(graph, ctx) {
874
+ return (
875
+ swapPrefillAttention(graph, ctx, {
876
+ from: 'attention_small_f16kv.wgsl',
877
+ to: 'attention_head256_f16kv.wgsl',
878
+ })
879
+ || swapPrefillAttention(graph, ctx, {
880
+ from: 'attention_streaming_f16kv.wgsl',
881
+ to: 'attention_head256_f16kv.wgsl',
882
+ })
883
+ );
884
+ }
885
+
393
886
  // =============================================================================
394
887
  // Transform: widenProjectionWeightsToF32
395
888
  // =============================================================================
@@ -414,6 +907,19 @@ const LAYER_PROJECTION_OPS = new Set([
414
907
  'gate_proj', 'up_proj', 'down_proj',
415
908
  ]);
416
909
 
910
+ const DENSE_Q4_PREFILL_FILES = new Set([
911
+ 'matmul_f16w_f32a.wgsl',
912
+ 'matmul_f16w_f32a_tiled.wgsl',
913
+ 'matmul_f16.wgsl',
914
+ 'matmul_f16_tiled.wgsl',
915
+ ]);
916
+
917
+ function resolveDensePrefillProjectionKernel(ctx) {
918
+ return ctx.activationDtype === 'f16'
919
+ ? 'matmul_f16.wgsl'
920
+ : 'matmul_f16w_f32a.wgsl';
921
+ }
922
+
417
923
  /**
418
924
  * Replace projection matmul kernels with f32 weight variants.
419
925
  *
@@ -472,6 +978,700 @@ export function widenProjectionWeightsToF32(graph, ctx) {
472
978
  return result;
473
979
  }
474
980
 
981
+ // =============================================================================
982
+ // Transform: remapDenseQ4KPrefillToQ4Native
983
+ // =============================================================================
984
+
985
+ /**
986
+ * Replace dense prefill projection kernels with Q4-native prefill variants.
987
+ *
988
+ * This applies only when the graph already exposes a compatible fused Q4 decode
989
+ * projection kernel. All prefill layer projections are remapped to the shared-A
990
+ * batched multicol Q4 prefill kernel so the transformed path remains valid for
991
+ * `M > 1` prefill workloads.
992
+ *
993
+ * Returns null when the graph does not have the required dense-prefill + Q4
994
+ * decode shape.
995
+ *
996
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
997
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
998
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
999
+ */
1000
+ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1001
+ const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
1002
+ if (!LAYER_PROJECTION_OPS.has(step[0])) {
1003
+ return false;
1004
+ }
1005
+ const entry = graph.kernels[step[1]];
1006
+ return entry != null && DENSE_Q4_PREFILL_FILES.has(entry.kernel);
1007
+ });
1008
+ if (densePrefillProjectionSteps.length === 0) {
1009
+ return null;
1010
+ }
1011
+
1012
+ const result = cloneGraph(graph);
1013
+ const existingSharedKey = findKernelKeyByFileAndEntry(
1014
+ result,
1015
+ 'fused_matmul_q4_batched_multicol_shared.wgsl',
1016
+ 'main'
1017
+ );
1018
+ let sharedKey = existingSharedKey;
1019
+ if (!sharedKey) {
1020
+ const q4DecodeKey = findPhaseKernelKey(
1021
+ graph,
1022
+ graph.decode || [],
1023
+ LAYER_PROJECTION_OPS,
1024
+ (entry) => entry.kernel === 'fused_matmul_q4.wgsl'
1025
+ );
1026
+ if (!q4DecodeKey) {
1027
+ return null;
1028
+ }
1029
+ const q4DecodeEntry = result.kernels[q4DecodeKey];
1030
+ sharedKey = deriveKernelKey(result.kernels, q4DecodeKey, '_prefill_shared');
1031
+ result.kernels[sharedKey] = deriveKernelEntry(
1032
+ q4DecodeEntry,
1033
+ 'fused_matmul_q4_batched_multicol_shared.wgsl',
1034
+ 'main',
1035
+ null
1036
+ );
1037
+ }
1038
+
1039
+ let changed = false;
1040
+ result.prefill = (result.prefill || []).map((step) => {
1041
+ const op = step[0];
1042
+ if (!LAYER_PROJECTION_OPS.has(op)) {
1043
+ return step;
1044
+ }
1045
+ const entry = result.kernels[step[1]];
1046
+ if (!entry || !DENSE_Q4_PREFILL_FILES.has(entry.kernel)) {
1047
+ return step;
1048
+ }
1049
+
1050
+ const replacementKey = sharedKey;
1051
+ if (replacementKey === step[1]) {
1052
+ return step;
1053
+ }
1054
+ changed = true;
1055
+ const next = [...step];
1056
+ next[1] = replacementKey;
1057
+ return next;
1058
+ });
1059
+
1060
+ return changed ? result : null;
1061
+ }
1062
+
1063
+ // =============================================================================
1064
+ // Transform: remapQ4KPrefillToDense
1065
+ // =============================================================================
1066
+
1067
+ /**
1068
+ * Replace fused Q4K prefill projection kernels with dense tiled variants.
1069
+ *
1070
+ * Decode remains unchanged so the runtime can keep using fused Q4K decode while
1071
+ * the loader exposes mixed dense+Q4K materializations for prefill.
1072
+ *
1073
+ * Returns null when the graph has no fused Q4K prefill projection kernels.
1074
+ *
1075
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1076
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1077
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1078
+ */
1079
+ export function remapQ4KPrefillToDense(graph, ctx) {
1080
+ const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
1081
+ if (!LAYER_PROJECTION_OPS.has(step[0])) {
1082
+ return false;
1083
+ }
1084
+ const entry = graph.kernels[step[1]];
1085
+ return entry != null && entry.kernel.startsWith('fused_matmul_q4');
1086
+ });
1087
+ if (q4PrefillProjectionSteps.length === 0) {
1088
+ return null;
1089
+ }
1090
+
1091
+ const denseKernelFile = resolveDensePrefillProjectionKernel(ctx);
1092
+ const result = cloneGraph(graph);
1093
+ let denseKey = findKernelKeyByFileAndEntry(result, denseKernelFile, 'main');
1094
+ if (!denseKey) {
1095
+ const sourceKey = q4PrefillProjectionSteps[0][1];
1096
+ const sourceEntry = result.kernels[sourceKey];
1097
+ denseKey = deriveKernelKey(result.kernels, sourceKey, '_prefill_dense');
1098
+ result.kernels[denseKey] = deriveKernelEntry(
1099
+ sourceEntry,
1100
+ denseKernelFile,
1101
+ 'main',
1102
+ null
1103
+ );
1104
+ }
1105
+
1106
+ let changed = false;
1107
+ result.prefill = (result.prefill || []).map((step) => {
1108
+ if (!LAYER_PROJECTION_OPS.has(step[0])) {
1109
+ return step;
1110
+ }
1111
+ const entry = result.kernels[step[1]];
1112
+ if (!entry || !entry.kernel.startsWith('fused_matmul_q4')) {
1113
+ return step;
1114
+ }
1115
+ if (step[1] === denseKey) {
1116
+ return step;
1117
+ }
1118
+ changed = true;
1119
+ const next = [...step];
1120
+ next[1] = denseKey;
1121
+ return next;
1122
+ });
1123
+
1124
+ return changed ? result : null;
1125
+ }
1126
+
1127
+ // =============================================================================
1128
+ // Transform: useLinearDecodeProjectionF16
1129
+ // =============================================================================
1130
+
1131
+ /**
1132
+ * Remap the linear-attention q_proj decode step onto the f16-activation fused
1133
+ * Q4 kernel for linear-attention layers only. Full-attention layers keep the
1134
+ * manifest-wide f32 activation contract.
1135
+ *
1136
+ * Only q_proj is remapped. o_proj is intentionally excluded: the o_proj
1137
+ * output enters the residual stream directly, and f16 truncation there
1138
+ * accumulates across the 18 linear-attention layers in the Qwen 3.5 pattern,
1139
+ * corrupting the logit distribution (empirically verified: degenerate
1140
+ * repetitive output under greedy decode). q_proj f16 is safe because the
1141
+ * linear attention core absorbs the f16 input into its f32 internal state.
1142
+ *
1143
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1144
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1145
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1146
+ */
1147
+ export function useLinearDecodeProjectionF16(graph, ctx) {
1148
+ const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
1149
+ if (!layerTypes || layerTypes.length === 0) {
1150
+ return null;
1151
+ }
1152
+
1153
+ const matchingLayers = layerTypes
1154
+ .map((layerType, layerIdx) => ({ layerType, layerIdx }))
1155
+ .filter(({ layerType }) => isLinearAttentionLayerType(layerType))
1156
+ .map(({ layerIdx }) => layerIdx);
1157
+ if (matchingLayers.length === 0) {
1158
+ return null;
1159
+ }
1160
+
1161
+ const result = cloneGraph(graph);
1162
+ const targetLayers = {
1163
+ allLayers: layerTypes.map((_, layerIdx) => layerIdx),
1164
+ matchingLayers,
1165
+ };
1166
+ const qProjIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1167
+ if (qProjIndex === -1) {
1168
+ return null;
1169
+ }
1170
+ const qProjStep = result.decode[qProjIndex];
1171
+ const qProjKernelKey = qProjStep[1];
1172
+ const qProjKernel = result.kernels[qProjKernelKey];
1173
+ if (!qProjKernel) {
1174
+ return null;
1175
+ }
1176
+
1177
+ const derivedEntry = deriveLinearDecodeF16KernelEntry(qProjKernel);
1178
+ if (!derivedEntry) {
1179
+ return null;
1180
+ }
1181
+
1182
+ const derivedKey = deriveKernelKey(result.kernels, qProjKernelKey, '_linear_f16out');
1183
+ result.kernels[derivedKey] = derivedEntry;
1184
+ const groupedEntries = buildGroupedLayerEntries(qProjStep, targetLayers, derivedKey);
1185
+ if (groupedEntries.length === 0) {
1186
+ return null;
1187
+ }
1188
+ result.decode = [
1189
+ ...result.decode.slice(0, qProjIndex),
1190
+ ...groupedEntries,
1191
+ ...result.decode.slice(qProjIndex + 1),
1192
+ ];
1193
+
1194
+ return result;
1195
+ }
1196
+
1197
+ // =============================================================================
1198
+ // Transform: remapQ4KDecodeToGemv
1199
+ // =============================================================================
1200
+
1201
+ /**
1202
+ * Replace fused Q4K decode projection kernels with GEMV subgroup variants.
1203
+ *
1204
+ * When Q4K weights have f16 materializations (mixed/dense loader mode), the
1205
+ * GEMV subgroup kernel on pre-dequantized f16 weights is significantly faster
1206
+ * than the fused Q4K kernel for M=1 decode (empirically 2.3x on Apple M-series).
1207
+ *
1208
+ * After this transform no decode kernels reference fused_matmul_q4*, which
1209
+ * signals the loader to use dense materialization (f16 only — no Q4K buffer
1210
+ * retained in GPU memory, reducing peak VRAM).
1211
+ *
1212
+ * Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
1213
+ * attention, residual, activation) are left untouched.
1214
+ *
1215
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1216
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1217
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1218
+ */
1219
+ export function remapQ4KDecodeToGemv(graph, ctx) {
1220
+ if (ctx.activationDtype === 'f16') {
1221
+ return null;
1222
+ }
1223
+
1224
+ const decodeSteps = graph.decode || [];
1225
+ const fusedDecodeKeys = new Set();
1226
+ for (const step of decodeSteps) {
1227
+ if (!Array.isArray(step)) continue;
1228
+ const kernelKey = step[1];
1229
+ const entry = graph.kernels[kernelKey];
1230
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
1231
+ fusedDecodeKeys.add(kernelKey);
1232
+ }
1233
+ }
1234
+ if (fusedDecodeKeys.size === 0) {
1235
+ return null;
1236
+ }
1237
+
1238
+ const result = cloneGraph(graph);
1239
+ const keyMap = new Map();
1240
+
1241
+ for (const key of fusedDecodeKeys) {
1242
+ const newKey = deriveKernelKey(result.kernels, key, '_gemv');
1243
+ result.kernels[newKey] = deriveKernelEntry(
1244
+ result.kernels[key],
1245
+ 'matmul_gemv_subgroup.wgsl',
1246
+ 'main_multicol',
1247
+ null
1248
+ );
1249
+ keyMap.set(key, newKey);
1250
+ }
1251
+
1252
+ result.decode = remapStepKeys(result.decode, keyMap);
1253
+ return result;
1254
+ }
1255
+
1256
+ // =============================================================================
1257
+ // Transform: remapQ4KDecodeAttentionToGemv (diagnostic)
1258
+ // =============================================================================
1259
+
1260
+ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
1261
+
1262
+ /**
1263
+ * Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
1264
+ * subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
1265
+ *
1266
+ * Diagnostic transform for isolating whether the GEMV correctness regression
1267
+ * originates in the attention or FFN projection path. Because FFN ops keep
1268
+ * their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
1269
+ * loader remains in mixed-materialization mode.
1270
+ *
1271
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1272
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1273
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1274
+ */
1275
+ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1276
+ if (ctx.activationDtype === 'f16') {
1277
+ return null;
1278
+ }
1279
+
1280
+ const decodeSteps = graph.decode || [];
1281
+ const attnFusedKeys = new Set();
1282
+ for (const step of decodeSteps) {
1283
+ if (!Array.isArray(step)) continue;
1284
+ const op = step[0];
1285
+ if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
1286
+ const kernelKey = step[1];
1287
+ const entry = graph.kernels[kernelKey];
1288
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
1289
+ attnFusedKeys.add(kernelKey);
1290
+ }
1291
+ }
1292
+ if (attnFusedKeys.size === 0) {
1293
+ return null;
1294
+ }
1295
+
1296
+ const result = cloneGraph(graph);
1297
+ const keyMap = new Map();
1298
+
1299
+ for (const key of attnFusedKeys) {
1300
+ const newKey = deriveKernelKey(result.kernels, key, '_attn_gemv');
1301
+ result.kernels[newKey] = deriveKernelEntry(
1302
+ result.kernels[key],
1303
+ 'matmul_gemv_subgroup.wgsl',
1304
+ 'main_multicol',
1305
+ null
1306
+ );
1307
+ keyMap.set(key, newKey);
1308
+ }
1309
+
1310
+ // Only remap attention projection steps, leave FFN steps unchanged.
1311
+ result.decode = result.decode.map((step) => {
1312
+ if (!Array.isArray(step)) return step;
1313
+ if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
1314
+ const replacement = keyMap.get(step[1]);
1315
+ if (replacement !== undefined) {
1316
+ const newStep = [...step];
1317
+ newStep[1] = replacement;
1318
+ return newStep;
1319
+ }
1320
+ return step;
1321
+ });
1322
+
1323
+ return result;
1324
+ }
1325
+
1326
+ // =============================================================================
1327
+ // Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
1328
+ // =============================================================================
1329
+
1330
+ /**
1331
+ * Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
1332
+ * optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
1333
+ * cooperative loading with fast nibble extraction for maximum M=1 throughput
1334
+ * while preserving full Q4K dequant precision (no f16 weight materialization).
1335
+ *
1336
+ * This is the production fix for the f16-precision-loss regression observed
1337
+ * when attention projections use the f16-weight GEMV path: softmax amplifies
1338
+ * the f16 round-trip error in Q/K/V projections, causing garbage output.
1339
+ * By keeping inline Q4K dequant (f32 arithmetic) the attention path stays
1340
+ * numerically correct. FFN projections are unaffected and can safely use
1341
+ * the f16-weight GEMV path via remapQ4KDecodeFFNToGemv.
1342
+ *
1343
+ * Because the derived kernel still references fused_matmul_q4.wgsl,
1344
+ * isKernelPathFusedQ4K stays true and the weight loader remains in
1345
+ * mixed-materialization mode (Q4K retained for attention, f16 for FFN).
1346
+ *
1347
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1348
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1349
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1350
+ */
1351
+ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
1352
+ if (ctx.activationDtype === 'f16') {
1353
+ return null;
1354
+ }
1355
+
1356
+ const decodeSteps = graph.decode || [];
1357
+ const attnFusedKeys = new Set();
1358
+ for (const step of decodeSteps) {
1359
+ if (!Array.isArray(step)) continue;
1360
+ const op = step[0];
1361
+ if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
1362
+ const kernelKey = step[1];
1363
+ const entry = graph.kernels[kernelKey];
1364
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
1365
+ attnFusedKeys.add(kernelKey);
1366
+ }
1367
+ }
1368
+ if (attnFusedKeys.size === 0) {
1369
+ return null;
1370
+ }
1371
+
1372
+ const result = cloneGraph(graph);
1373
+ const keyMap = new Map();
1374
+
1375
+ for (const key of attnFusedKeys) {
1376
+ const newKey = deriveKernelKey(result.kernels, key, '_gemv');
1377
+ result.kernels[newKey] = deriveKernelEntry(
1378
+ result.kernels[key],
1379
+ 'fused_matmul_q4.wgsl',
1380
+ 'main_gemv',
1381
+ null
1382
+ );
1383
+ keyMap.set(key, newKey);
1384
+ }
1385
+
1386
+ // Only remap attention projection steps, leave FFN steps unchanged.
1387
+ result.decode = result.decode.map((step) => {
1388
+ if (!Array.isArray(step)) return step;
1389
+ if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
1390
+ const replacement = keyMap.get(step[1]);
1391
+ if (replacement !== undefined) {
1392
+ const newStep = [...step];
1393
+ newStep[1] = replacement;
1394
+ return newStep;
1395
+ }
1396
+ return step;
1397
+ });
1398
+
1399
+ return result;
1400
+ }
1401
+
1402
+ // =============================================================================
1403
+ // Transform: remapQ4KDecodeFFNToGemv (diagnostic)
1404
+ // =============================================================================
1405
+
1406
+ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
1407
+
1408
+ /**
1409
+ * Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
1410
+ * variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
1411
+ *
1412
+ * Diagnostic complement to `remapQ4KDecodeAttentionToGemv`. Together these
1413
+ * two transforms isolate whether the GEMV decode regression originates in
1414
+ * the attention or FFN projection path. Because attention ops keep their
1415
+ * fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
1416
+ * remains in mixed-materialization mode.
1417
+ *
1418
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1419
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1420
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1421
+ */
1422
+ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1423
+ if (ctx.activationDtype === 'f16') {
1424
+ return null;
1425
+ }
1426
+
1427
+ const decodeSteps = graph.decode || [];
1428
+ const ffnFusedKeys = new Set();
1429
+ for (const step of decodeSteps) {
1430
+ if (!Array.isArray(step)) continue;
1431
+ const op = step[0];
1432
+ if (!FFN_PROJECTION_OPS.has(op)) continue;
1433
+ const kernelKey = step[1];
1434
+ const entry = graph.kernels[kernelKey];
1435
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
1436
+ ffnFusedKeys.add(kernelKey);
1437
+ }
1438
+ }
1439
+ if (ffnFusedKeys.size === 0) {
1440
+ return null;
1441
+ }
1442
+
1443
+ const result = cloneGraph(graph);
1444
+ const keyMap = new Map();
1445
+
1446
+ for (const key of ffnFusedKeys) {
1447
+ const newKey = deriveKernelKey(result.kernels, key, '_ffn_gemv');
1448
+ result.kernels[newKey] = deriveKernelEntry(
1449
+ result.kernels[key],
1450
+ 'matmul_gemv_subgroup.wgsl',
1451
+ 'main_multicol',
1452
+ null
1453
+ );
1454
+ keyMap.set(key, newKey);
1455
+ }
1456
+
1457
+ // Only remap FFN projection steps, leave attention steps unchanged.
1458
+ result.decode = result.decode.map((step) => {
1459
+ if (!Array.isArray(step)) return step;
1460
+ if (!FFN_PROJECTION_OPS.has(step[0])) return step;
1461
+ const replacement = keyMap.get(step[1]);
1462
+ if (replacement !== undefined) {
1463
+ const newStep = [...step];
1464
+ newStep[1] = replacement;
1465
+ return newStep;
1466
+ }
1467
+ return step;
1468
+ });
1469
+
1470
+ return result;
1471
+ }
1472
+
1473
+ // =============================================================================
1474
+ // Transform: useQwenDecodeF16Matmuls
1475
+ // =============================================================================
1476
+
1477
+ /**
1478
+ * Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
1479
+ * kernels while keeping the manifest-wide f32 activation contract intact.
1480
+ *
1481
+ * This transform is intentionally selective:
1482
+ * - FFN gate/up decode matmuls switch to f16a so decode can bypass the slow
1483
+ * fused-q4k FFN path when capability policy opts in.
1484
+ * - LM head decode switches to the subgroup f16a GEMV path.
1485
+ *
1486
+ * FFN down remains on the f32-output contract so the layer residual path stays
1487
+ * numerically aligned with the manifest-owned activation dtype.
1488
+ *
1489
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1490
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1491
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1492
+ */
1493
+ export function useQwenDecodeF16Matmuls(graph, ctx) {
1494
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1495
+ if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
1496
+ return null;
1497
+ }
1498
+
1499
+ const result = cloneGraph(graph);
1500
+ let changed = false;
1501
+
1502
+ for (const op of ['gate_proj', 'up_proj']) {
1503
+ const stepIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === op);
1504
+ if (stepIndex === -1) {
1505
+ continue;
1506
+ }
1507
+ const step = result.decode[stepIndex];
1508
+ const kernelKey = step[1];
1509
+ const kernelEntry = result.kernels[kernelKey];
1510
+ if (!kernelEntry) {
1511
+ continue;
1512
+ }
1513
+ const derivedEntry = deriveLinearDecodeF16KernelEntry(kernelEntry);
1514
+ if (!derivedEntry) {
1515
+ continue;
1516
+ }
1517
+ const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_decode_f16out');
1518
+ result.kernels[derivedKey] = derivedEntry;
1519
+ const replacement = [...step];
1520
+ replacement[1] = derivedKey;
1521
+ result.decode = [
1522
+ ...result.decode.slice(0, stepIndex),
1523
+ replacement,
1524
+ ...result.decode.slice(stepIndex + 1),
1525
+ ];
1526
+ changed = true;
1527
+ }
1528
+
1529
+ const postLayerResult = replacePhaseStepKernelKey(
1530
+ result.postLayer ?? [],
1531
+ 'lm_head',
1532
+ (() => {
1533
+ const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
1534
+ if (!lmHeadStep) {
1535
+ return null;
1536
+ }
1537
+ const lmHeadKernelKey = lmHeadStep[1];
1538
+ const lmHeadKernel = result.kernels[lmHeadKernelKey];
1539
+ if (!lmHeadKernel) {
1540
+ return null;
1541
+ }
1542
+ const derivedEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
1543
+ if (!derivedEntry) {
1544
+ return null;
1545
+ }
1546
+ const derivedKey = deriveKernelKey(result.kernels, lmHeadKernelKey, '_decode_f16out');
1547
+ result.kernels[derivedKey] = derivedEntry;
1548
+ return derivedKey;
1549
+ })()
1550
+ );
1551
+ if (postLayerResult.changed) {
1552
+ result.postLayer = postLayerResult.steps;
1553
+ changed = true;
1554
+ }
1555
+
1556
+ return changed ? result : null;
1557
+ }
1558
+
1559
+ // =============================================================================
1560
+ // Transform: useQwenF16PrimaryMatmuls
1561
+ // =============================================================================
1562
+
1563
+ /**
1564
+ * Promote the Qwen 3.5 0.8B execution graph onto its fast f16 primary lane
1565
+ * when the runtime explicitly requests f16 activations.
1566
+ *
1567
+ * This transform narrows the hot matmul path while keeping `o_proj` on the
1568
+ * stable manifest-owned f32-output kernel. The residual stream feeds directly
1569
+ * into the next RMSNorm, and the Qwen 3.5 0.8B promoted f16 lane becomes
1570
+ * numerically unstable when `o_proj` writes f16 there (observed first at the
1571
+ * first full-attention block's post-attention RMSNorm).
1572
+ *
1573
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1574
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1575
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1576
+ */
1577
+ export function useQwenF16PrimaryMatmuls(graph, ctx) {
1578
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1579
+ if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
1580
+ return null;
1581
+ }
1582
+
1583
+ const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
1584
+ if (!layerTypes || layerTypes.length === 0) {
1585
+ return null;
1586
+ }
1587
+
1588
+ const result = cloneGraph(graph);
1589
+ let changed = false;
1590
+
1591
+ for (const [phaseName, op] of [['decode', 'attention'], ['prefill', 'attention']]) {
1592
+ const phaseSteps = result[phaseName] || [];
1593
+ const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
1594
+ const kernelKey = step?.[1];
1595
+ const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1596
+ const derivedEntry = deriveF16AttentionKernelEntry(kernelEntry);
1597
+ if (!step || !kernelKey || !derivedEntry) {
1598
+ continue;
1599
+ }
1600
+ const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f16');
1601
+ result.kernels[derivedKey] = derivedEntry;
1602
+ const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, derivedKey);
1603
+ if (phaseResult.changed) {
1604
+ result[phaseName] = phaseResult.steps;
1605
+ changed = true;
1606
+ }
1607
+ }
1608
+
1609
+ const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1610
+ const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
1611
+ const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(decodeProjectionKernel);
1612
+ if (decodeProjectionStep && decodeProjectionEntry) {
1613
+ const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_primary_f16');
1614
+ result.kernels[decodeProjectionKey] = decodeProjectionEntry;
1615
+ for (const op of ['q_proj', 'k_proj', 'v_proj', 'gate_proj', 'up_proj']) {
1616
+ const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
1617
+ if (phaseResult.changed) {
1618
+ result.decode = phaseResult.steps;
1619
+ changed = true;
1620
+ }
1621
+ }
1622
+ }
1623
+
1624
+ const prefillProjectionStep = (result.prefill || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1625
+ const prefillProjectionKernel = prefillProjectionStep ? result.kernels[prefillProjectionStep[1]] : null;
1626
+ const prefillProjectionEntry = deriveQ4PrefillF16KernelEntry(prefillProjectionKernel);
1627
+ if (prefillProjectionStep && prefillProjectionEntry) {
1628
+ const prefillProjectionKey = deriveKernelKey(result.kernels, prefillProjectionStep[1], '_primary_f16');
1629
+ result.kernels[prefillProjectionKey] = prefillProjectionEntry;
1630
+ for (const op of ['q_proj', 'k_proj', 'v_proj']) {
1631
+ const phaseResult = replacePhaseStepKernelKey(result.prefill, op, prefillProjectionKey);
1632
+ if (phaseResult.changed) {
1633
+ result.prefill = phaseResult.steps;
1634
+ changed = true;
1635
+ }
1636
+ }
1637
+ }
1638
+
1639
+ for (const [phaseName, op] of [['decode', 'o_proj'], ['prefill', 'o_proj']]) {
1640
+ const phaseSteps = result[phaseName] || [];
1641
+ const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
1642
+ const kernelKey = step?.[1];
1643
+ const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1644
+ if (!step || !kernelKey || !kernelEntry) {
1645
+ continue;
1646
+ }
1647
+ const boundaryKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f32_boundary');
1648
+ result.kernels[boundaryKey] = deriveKernelEntryWithPrecision(kernelEntry, {
1649
+ inputDtype: 'f32',
1650
+ outputDtype: 'f32',
1651
+ });
1652
+ const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, boundaryKey);
1653
+ if (phaseResult.changed) {
1654
+ result[phaseName] = phaseResult.steps;
1655
+ changed = true;
1656
+ }
1657
+ }
1658
+
1659
+ const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
1660
+ const lmHeadKernel = lmHeadStep ? result.kernels[lmHeadStep[1]] : null;
1661
+ const lmHeadEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
1662
+ if (lmHeadStep && lmHeadEntry) {
1663
+ const lmHeadKey = deriveKernelKey(result.kernels, lmHeadStep[1], '_primary_f16');
1664
+ result.kernels[lmHeadKey] = lmHeadEntry;
1665
+ const phaseResult = replacePhaseStepKernelKey(result.postLayer, 'lm_head', lmHeadKey);
1666
+ if (phaseResult.changed) {
1667
+ result.postLayer = phaseResult.steps;
1668
+ changed = true;
1669
+ }
1670
+ }
1671
+
1672
+ return changed ? result : null;
1673
+ }
1674
+
475
1675
  // =============================================================================
476
1676
  // Composition
477
1677
  // =============================================================================
@@ -504,9 +1704,21 @@ export function composeTransforms(...transforms) {
504
1704
 
505
1705
  /** @type {Readonly<Record<string, Function>>} */
506
1706
  export const TRANSFORMS = Object.freeze({
1707
+ narrowToF16Activations,
507
1708
  removeSubgroups,
508
1709
  widenToF32Activations,
1710
+ widenToF32CorrectnessFallback,
509
1711
  swapPrefillAttention,
1712
+ useHead256PrefillAttention,
510
1713
  widenProjectionWeightsToF32,
1714
+ remapDenseQ4KPrefillToQ4Native,
1715
+ remapQ4KPrefillToDense,
1716
+ useLinearDecodeProjectionF16,
1717
+ remapQ4KDecodeToGemv,
1718
+ remapQ4KDecodeAttentionToGemv,
1719
+ remapQ4KDecodeAttentionToFusedQ4KGemv,
1720
+ remapQ4KDecodeFFNToGemv,
1721
+ useQwenF16PrimaryMatmuls,
1722
+ useQwenDecodeF16Matmuls,
511
1723
  composeTransforms,
512
1724
  });