doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -13,16 +13,14 @@
13
13
  // Helpers
14
14
  // =============================================================================
15
15
 
16
- /**
16
+ /*
17
17
  * Deep-clone an execution graph.
18
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
19
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph}
20
18
  */
21
19
  function cloneGraph(graph) {
22
20
  return structuredClone(graph);
23
21
  }
24
22
 
25
- /**
23
+ /*
26
24
  * Shader files that require subgroups even though "subgroup" is not in the filename.
27
25
  * Online attention kernels use subgroup reductions internally.
28
26
  */
@@ -35,21 +33,24 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
35
33
  ['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
36
34
  ['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
37
35
  ['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
36
+ ['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
38
37
  ['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
39
38
  ['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
39
+ ['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
40
40
  ['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
41
41
  ['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
42
42
  ['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
43
+ ['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
43
44
  ['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
44
45
  ['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
45
46
  ['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
46
47
  ['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
48
+ ['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
49
+ ['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
47
50
  ]);
48
51
 
49
- /**
52
+ /*
50
53
  * Check whether a kernel entry requires subgroup support.
51
- * @param {{ kernel: string }} kernelEntry
52
- * @returns {boolean}
53
54
  */
54
55
  function isSubgroupKernel(kernelEntry) {
55
56
  if (typeof kernelEntry.kernel !== 'string') return false;
@@ -61,11 +62,8 @@ function requiresNoSubgroupFallback(kernelEntry) {
61
62
  return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
62
63
  }
63
64
 
64
- /**
65
+ /*
65
66
  * Find all kernel keys in the graph whose `kernel` file matches the given filename.
66
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
67
- * @param {string} filename
68
- * @returns {string[]}
69
67
  */
70
68
  function findKernelKeysByFile(graph, filename) {
71
69
  const keys = [];
@@ -77,23 +75,15 @@ function findKernelKeysByFile(graph, filename) {
77
75
  return keys;
78
76
  }
79
77
 
80
- /**
78
+ /*
81
79
  * Check whether any kernel in the graph uses the given shader file.
82
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
83
- * @param {string} filename
84
- * @returns {boolean}
85
80
  */
86
81
  function hasKernelFile(graph, filename) {
87
82
  return findKernelKeysByFile(graph, filename).length > 0;
88
83
  }
89
84
 
90
- /**
85
+ /*
91
86
  * Create a new kernel entry with the digest cleared (shader changed).
92
- * @param {object} base - original kernel entry
93
- * @param {string} newFile - new shader filename
94
- * @param {string} newEntry - new entry point name
95
- * @param {object|null} [constants] - override constants (null to remove)
96
- * @returns {object}
97
87
  */
98
88
  function deriveKernelEntry(base, newFile, newEntry, constants) {
99
89
  const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
@@ -111,6 +101,11 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
111
101
  return derived;
112
102
  }
113
103
 
104
+ function deriveMappedKernelEntry(base, newFile) {
105
+ const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
106
+ return deriveKernelEntry(base, newFile, newEntry);
107
+ }
108
+
114
109
  function deriveKernelPrecision(base, newFile) {
115
110
  const precision = base.precision ? { ...base.precision } : {};
116
111
  const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
@@ -135,12 +130,8 @@ function deriveKernelPrecision(base, newFile) {
135
130
  return precision;
136
131
  }
137
132
 
138
- /**
133
+ /*
139
134
  * Derive a non-colliding kernel key name.
140
- * @param {object} kernels - existing kernels dict
141
- * @param {string} baseKey - original key
142
- * @param {string} suffix - suffix to append
143
- * @returns {string}
144
135
  */
145
136
  function deriveKernelKey(kernels, baseKey, suffix) {
146
137
  const candidate = `${baseKey}${suffix}`;
@@ -154,11 +145,8 @@ function deriveKernelKey(kernels, baseKey, suffix) {
154
145
  return `${candidate}_${counter}`;
155
146
  }
156
147
 
157
- /**
148
+ /*
158
149
  * Replace kernel key references in step tuples.
159
- * @param {Array<Array>} steps
160
- * @param {Map<string, string>} keyMap - oldKey → newKey
161
- * @returns {Array<Array>}
162
150
  */
163
151
  function remapStepKeys(steps, keyMap) {
164
152
  return steps.map((step) => {
@@ -173,12 +161,8 @@ function remapStepKeys(steps, keyMap) {
173
161
  });
174
162
  }
175
163
 
176
- /**
164
+ /*
177
165
  * Check whether a step tuple's kernel key resolves to the given shader file.
178
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
179
- * @param {Array} step
180
- * @param {string} filename
181
- * @returns {boolean}
182
166
  */
183
167
  function stepUsesFile(graph, step, filename) {
184
168
  const kernelKey = step[1];
@@ -186,14 +170,9 @@ function stepUsesFile(graph, step, filename) {
186
170
  return entry != null && entry.kernel === filename;
187
171
  }
188
172
 
189
- /**
173
+ /*
190
174
  * Find the first kernel key used by matching ops in a phase whose shader file
191
175
  * satisfies the provided predicate.
192
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
193
- * @param {Array<Array>} steps
194
- * @param {ReadonlySet<string>} ops
195
- * @param {(entry: { kernel: string, entry: string }) => boolean} predicate
196
- * @returns {string | null}
197
176
  */
198
177
  function findPhaseKernelKey(graph, steps, ops, predicate) {
199
178
  for (const step of steps || []) {
@@ -208,12 +187,8 @@ function findPhaseKernelKey(graph, steps, ops, predicate) {
208
187
  return null;
209
188
  }
210
189
 
211
- /**
190
+ /*
212
191
  * Find an existing kernel key by shader file and entry point.
213
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
214
- * @param {string} filename
215
- * @param {string} entryPoint
216
- * @returns {string | null}
217
192
  */
218
193
  function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
219
194
  for (const [key, entry] of Object.entries(graph.kernels)) {
@@ -292,6 +267,28 @@ function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
292
267
  return { steps: nextSteps, changed };
293
268
  }
294
269
 
270
+ function findPhaseStep(steps, op) {
271
+ if (!Array.isArray(steps) || !op) {
272
+ return null;
273
+ }
274
+ for (const entry of steps) {
275
+ if (Array.isArray(entry)) {
276
+ if (entry[0] === op) {
277
+ return entry;
278
+ }
279
+ continue;
280
+ }
281
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
282
+ continue;
283
+ }
284
+ const nested = findPhaseStep(entry.steps, op);
285
+ if (nested) {
286
+ return nested;
287
+ }
288
+ }
289
+ return null;
290
+ }
291
+
295
292
  function deriveKernelEntryWithPrecision(base, precision) {
296
293
  return {
297
294
  ...base,
@@ -307,6 +304,7 @@ const ATTENTION_F16KV_TO_F16_MAP = new Map([
307
304
  ['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
308
305
  ['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
309
306
  ['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
307
+ ['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
310
308
  ]);
311
309
 
312
310
  function deriveF16AttentionKernelEntry(base) {
@@ -366,6 +364,29 @@ function deriveLmHeadDecodeF16KernelEntry(base) {
366
364
  return null;
367
365
  }
368
366
 
367
+ function deriveDenseDecodeF16KernelEntry(base) {
368
+ if (typeof base?.kernel !== 'string') {
369
+ return null;
370
+ }
371
+ const precision = {
372
+ inputDtype: 'f16',
373
+ outputDtype: 'f16',
374
+ };
375
+ if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
376
+ return {
377
+ ...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
378
+ precision: {
379
+ ...(base.precision ?? {}),
380
+ ...precision,
381
+ },
382
+ };
383
+ }
384
+ if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
385
+ return deriveKernelEntryWithPrecision(base, precision);
386
+ }
387
+ return null;
388
+ }
389
+
369
390
  function deriveQ4DecodeF16KernelEntry(base) {
370
391
  if (typeof base?.kernel !== 'string') {
371
392
  return null;
@@ -400,6 +421,9 @@ function deriveQ4PrefillF16KernelEntry(base) {
400
421
  inputDtype: 'f16',
401
422
  outputDtype: 'f16',
402
423
  };
424
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
425
+ return deriveKernelEntryWithPrecision(base, precision);
426
+ }
403
427
  if (base.kernel.startsWith('fused_matmul_q4_batched')) {
404
428
  return {
405
429
  ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
@@ -412,6 +436,53 @@ function deriveQ4PrefillF16KernelEntry(base) {
412
436
  return null;
413
437
  }
414
438
 
439
+ function deriveQ4WideTilePrefillF16KernelEntry(base) {
440
+ if (typeof base?.kernel !== 'string') {
441
+ return null;
442
+ }
443
+ const precision = {
444
+ inputDtype: 'f16',
445
+ outputDtype: 'f16',
446
+ };
447
+ if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
448
+ return {
449
+ ...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
450
+ precision: {
451
+ ...(base.precision ?? {}),
452
+ ...precision,
453
+ },
454
+ };
455
+ }
456
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
457
+ return deriveKernelEntryWithPrecision(base, precision);
458
+ }
459
+ return null;
460
+ }
461
+
462
+ function deriveQ4PrefillF16AccumKernelEntry(base) {
463
+ if (typeof base?.kernel !== 'string') {
464
+ return null;
465
+ }
466
+ const precision = {
467
+ inputDtype: 'f16',
468
+ outputDtype: 'f16',
469
+ };
470
+ if (
471
+ base.kernel === 'fused_matmul_q4_widetile.wgsl'
472
+ || base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
473
+ || base.kernel.startsWith('fused_matmul_q4_batched')
474
+ ) {
475
+ return {
476
+ ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
477
+ precision: {
478
+ ...(base.precision ?? {}),
479
+ ...precision,
480
+ },
481
+ };
482
+ }
483
+ return null;
484
+ }
485
+
415
486
  function deriveQ4DecodeF32ActivationKernelEntry(base) {
416
487
  if (typeof base?.kernel !== 'string') {
417
488
  return null;
@@ -438,16 +509,38 @@ function deriveQ4DecodeF32ActivationKernelEntry(base) {
438
509
  return null;
439
510
  }
440
511
 
441
- function deriveQ4PrefillF32ActivationKernelEntry(base) {
512
+ function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
442
513
  if (typeof base?.kernel !== 'string') {
443
514
  return null;
444
515
  }
516
+ const replacement = options.fullF32 === true
517
+ ? {
518
+ kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
519
+ entry: 'main',
520
+ }
521
+ : {
522
+ kernel: 'fused_matmul_q4_widetile.wgsl',
523
+ entry: 'main',
524
+ };
525
+ if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
526
+ return deriveKernelEntryWithPrecision(
527
+ deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
528
+ { inputDtype: 'f32', outputDtype: 'f32' }
529
+ );
530
+ }
445
531
  if (
446
532
  base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
447
533
  && base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
534
+ && base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
448
535
  ) {
449
536
  return null;
450
537
  }
538
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
539
+ return deriveKernelEntryWithPrecision(
540
+ deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
541
+ { inputDtype: 'f32', outputDtype: 'f32' }
542
+ );
543
+ }
451
544
  return deriveKernelEntryWithPrecision(
452
545
  deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
453
546
  { inputDtype: 'f32', outputDtype: 'f32' }
@@ -476,15 +569,12 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
476
569
  // Transform: removeSubgroups
477
570
  // =============================================================================
478
571
 
479
- /**
572
+ /*
480
573
  * Remove subgroup shader dependencies from decode and postLayer steps.
481
574
  * Prefill steps are left untouched (they already use tiled matmul).
482
575
  *
483
576
  * Returns null if the graph has no subgroup kernels.
484
577
  *
485
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
486
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
487
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
488
578
  */
489
579
  export function removeSubgroups(graph, ctx) {
490
580
  const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
@@ -571,19 +661,21 @@ export function removeSubgroups(graph, ctx) {
571
661
  // Transform: widenToF32Activations
572
662
  // =============================================================================
573
663
 
574
- /**
664
+ /*
575
665
  * Activation-only widening: f16-activation shaders → f32-activation variants
576
666
  * that still use f16 for weights and KV cache. Requires shader-f16 for weight
577
667
  * and KV buffer reads.
578
- * @type {ReadonlyMap<string, string>}
579
668
  */
580
669
  const F16_TO_F32_ACTIVATION_MAP = new Map([
581
670
  ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
582
671
  ['rope_f16.wgsl', 'rope.wgsl'],
583
672
  ['residual_f16.wgsl', 'residual.wgsl'],
584
673
  ['gelu_f16.wgsl', 'gelu.wgsl'],
674
+ ['silu_f16.wgsl', 'silu.wgsl'],
585
675
  ['sample_f16.wgsl', 'sample.wgsl'],
586
676
  ['gather_f16.wgsl', 'gather.wgsl'],
677
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
678
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
587
679
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
588
680
  ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
589
681
  ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
@@ -591,20 +683,25 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
591
683
  ['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
592
684
  ['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
593
685
  ['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
686
+ ['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
594
687
  ]);
595
688
 
596
- /**
689
+ /*
597
690
  * Activation-only narrowing: f32-activation shaders that still consume f16
598
691
  * weights/KV are rewritten onto the matching f16-activation lane.
599
692
  *
600
693
  * This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
601
694
  * runtime session explicitly requests f16 activations for an execution-v1
602
695
  * graph that was authored with conservative f32 activation defaults.
603
- * @type {ReadonlyMap<string, string>}
604
696
  */
605
697
  const F32_TO_F16_ACTIVATION_MAP = new Map(
606
698
  Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
607
699
  );
700
+ F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
701
+ F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
702
+ // head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
703
+ // f16 requests fail-closed until that path has its own evidence.
704
+ F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
608
705
 
609
706
  function hasExplicitF32ActivationContract(entry) {
610
707
  const precision = entry?.precision;
@@ -616,37 +713,38 @@ function hasExplicitF32ActivationContract(entry) {
616
713
  || precision.outputDtype === 'f32';
617
714
  }
618
715
 
619
- /**
716
+ /*
620
717
  * Correctness fallback: preserve f16 weights where possible, but widen both
621
718
  * activations and KV-cache interactions onto the stable f32 execution lane.
622
719
  * Used for alternate-plan recovery after finiteness failure.
623
- * @type {ReadonlyMap<string, string>}
624
720
  */
625
721
  const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
626
722
  ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
627
723
  ['rope_f16.wgsl', 'rope.wgsl'],
628
724
  ['residual_f16.wgsl', 'residual.wgsl'],
629
725
  ['gelu_f16.wgsl', 'gelu.wgsl'],
726
+ ['silu_f16.wgsl', 'silu.wgsl'],
630
727
  ['sample_f16.wgsl', 'sample.wgsl'],
631
728
  ['gather_f16.wgsl', 'gather.wgsl'],
729
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
730
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
632
731
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
633
732
  ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
634
733
  ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
635
- ['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
636
- ['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
734
+ ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
735
+ ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
637
736
  ['attention_small_f16.wgsl', 'attention_small.wgsl'],
638
737
  ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
639
- ['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
640
- ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
738
+ ['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
739
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
641
740
  ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
642
741
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
643
742
  ]);
644
743
 
645
- /**
744
+ /*
646
745
  * Full f32 widening: every shader that uses `enable f16;` is replaced with a
647
746
  * pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
648
747
  * Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
649
- * @type {ReadonlyMap<string, string>}
650
748
  */
651
749
  const FULL_F32_SHADER_MAP = new Map([
652
750
  // f16-activation utility kernels → f32
@@ -654,8 +752,11 @@ const FULL_F32_SHADER_MAP = new Map([
654
752
  ['rope_f16.wgsl', 'rope.wgsl'],
655
753
  ['residual_f16.wgsl', 'residual.wgsl'],
656
754
  ['gelu_f16.wgsl', 'gelu.wgsl'],
755
+ ['silu_f16.wgsl', 'silu.wgsl'],
657
756
  ['sample_f16.wgsl', 'sample.wgsl'],
658
757
  ['gather_f16.wgsl', 'gather.wgsl'],
758
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
759
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
659
760
  // f16-activation matmul → f32
660
761
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
661
762
  ['matmul_f16.wgsl', 'matmul_f32.wgsl'],
@@ -665,19 +766,19 @@ const FULL_F32_SHADER_MAP = new Map([
665
766
  ['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
666
767
  ['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
667
768
  // f16-activation attention → f32
668
- ['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
669
- ['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
769
+ ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
770
+ ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
670
771
  ['attention_small_f16.wgsl', 'attention_small.wgsl'],
671
772
  ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
672
773
  // f16kv attention (f32 Q, f16 KV) → f32
673
- ['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
674
- ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
774
+ ['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
775
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
675
776
  ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
676
777
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
677
778
  ['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
678
779
  ]);
679
780
 
680
- /**
781
+ /*
681
782
  * Widen all f16-activation shaders to f32-activation equivalents.
682
783
  *
683
784
  * Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
@@ -686,9 +787,6 @@ const FULL_F32_SHADER_MAP = new Map([
686
787
  * NOTE: The caller is responsible for also updating session.activationDtype
687
788
  * to reflect the widened dtype.
688
789
  *
689
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
690
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
691
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
692
790
  */
693
791
  export function widenToF32Activations(graph, ctx) {
694
792
  // Bail out if fused f16 FFN is present — no direct f32 equivalent
@@ -702,9 +800,12 @@ export function widenToF32Activations(graph, ctx) {
702
800
  const shaderMap = ctx.capabilities?.hasF16 === false
703
801
  ? FULL_F32_SHADER_MAP
704
802
  : F16_TO_F32_ACTIVATION_MAP;
803
+ const fullF32 = ctx.capabilities?.hasF16 === false;
705
804
 
706
805
  const hasTargetShader = Object.values(graph.kernels).some(
707
806
  (entry) => shaderMap.has(entry.kernel)
807
+ || deriveQ4DecodeF32ActivationKernelEntry(entry)
808
+ || deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
708
809
  );
709
810
  if (!hasTargetShader) {
710
811
  return null;
@@ -714,27 +815,24 @@ export function widenToF32Activations(graph, ctx) {
714
815
 
715
816
  for (const [key, entry] of Object.entries(result.kernels)) {
716
817
  const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
717
- ?? deriveQ4PrefillF32ActivationKernelEntry(entry);
818
+ ?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
718
819
  if (q4FallbackEntry) {
719
820
  result.kernels[key] = q4FallbackEntry;
720
821
  continue;
721
822
  }
722
823
  const replacement = shaderMap.get(entry.kernel);
723
824
  if (replacement !== undefined) {
724
- result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
825
+ result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
725
826
  }
726
827
  }
727
828
 
728
829
  return result;
729
830
  }
730
831
 
731
- /**
832
+ /*
732
833
  * Widen an f16 execution graph onto the stable f32 correctness lane used for
733
834
  * alternate-plan recovery after finiteness failure.
734
835
  *
735
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
736
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
737
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
738
836
  */
739
837
  export function widenToF32CorrectnessFallback(graph, ctx) {
740
838
  if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
@@ -758,22 +856,19 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
758
856
  }
759
857
  const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
760
858
  if (replacement !== undefined) {
761
- result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
859
+ result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
762
860
  }
763
861
  }
764
862
  return result;
765
863
  }
766
864
 
767
- /**
865
+ /*
768
866
  * Narrow f32-activation shaders back onto their f16-activation equivalents.
769
867
  *
770
868
  * Returns null if the graph has no supported f32-activation kernels to swap or
771
869
  * if the runtime did not explicitly request f16 activations on an f16-capable
772
870
  * GPU.
773
871
  *
774
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
775
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
776
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
777
872
  */
778
873
  export function narrowToF16Activations(graph, ctx) {
779
874
  if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
@@ -804,7 +899,6 @@ export function narrowToF16Activations(graph, ctx) {
804
899
  // Transform: swapPrefillAttention
805
900
  // =============================================================================
806
901
 
807
- /** @type {ReadonlyMap<string, string>} */
808
902
  const PREFILL_ATTENTION_PAIRS = new Map([
809
903
  ['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
810
904
  ['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
@@ -812,7 +906,27 @@ const PREFILL_ATTENTION_PAIRS = new Map([
812
906
  ['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
813
907
  ]);
814
908
 
815
- /**
909
+ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
910
+ for (const entry of graph.prefill || []) {
911
+ if (Array.isArray(entry)) {
912
+ if (entry[1] === kernelKey) {
913
+ return true;
914
+ }
915
+ continue;
916
+ }
917
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
918
+ continue;
919
+ }
920
+ for (const step of entry.steps) {
921
+ if (Array.isArray(step) && step[1] === kernelKey) {
922
+ return true;
923
+ }
924
+ }
925
+ }
926
+ return false;
927
+ }
928
+
929
+ /*
816
930
  * Swap prefill attention kernel between streaming and small variants.
817
931
  *
818
932
  * The `opts` parameter specifies the direction:
@@ -821,10 +935,6 @@ const PREFILL_ATTENTION_PAIRS = new Map([
821
935
  * If `from`/`to` are not provided, uses the bidirectional pair map.
822
936
  * Returns null if no matching prefill attention kernel is found.
823
937
  *
824
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
825
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
826
- * @param {{ from?: string, to?: string }} [opts]
827
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
828
938
  */
829
939
  export function swapPrefillAttention(graph, ctx, opts) {
830
940
  const from = opts?.from;
@@ -847,8 +957,7 @@ export function swapPrefillAttention(graph, ctx, opts) {
847
957
  }
848
958
 
849
959
  if (target !== undefined) {
850
- // Verify this kernel is actually used in a prefill step
851
- const usedInPrefill = (graph.prefill || []).some((step) => step[1] === key);
960
+ const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
852
961
  if (usedInPrefill) {
853
962
  result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
854
963
  changed = true;
@@ -863,31 +972,47 @@ export function swapPrefillAttention(graph, ctx, opts) {
863
972
  // Transform: useHead256PrefillAttention
864
973
  // =============================================================================
865
974
 
866
- /**
975
+ /*
976
+ * Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
977
+ *
978
+ */
979
+ export function useHead256SmallPrefillAttention(graph, ctx) {
980
+ return swapPrefillAttention(graph, ctx, {
981
+ from: 'attention_small_f16kv.wgsl',
982
+ to: 'attention_head256_f16kv.wgsl',
983
+ });
984
+ }
985
+
986
+ /*
867
987
  * Promote prefill attention onto the fixed 256-dim shared-block kernel.
868
988
  *
869
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
870
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
871
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
872
989
  */
873
990
  export function useHead256PrefillAttention(graph, ctx) {
874
- return (
875
- swapPrefillAttention(graph, ctx, {
876
- from: 'attention_small_f16kv.wgsl',
877
- to: 'attention_head256_f16kv.wgsl',
878
- })
879
- || swapPrefillAttention(graph, ctx, {
880
- from: 'attention_streaming_f16kv.wgsl',
881
- to: 'attention_head256_f16kv.wgsl',
882
- })
883
- );
991
+ let current = graph;
992
+ let changed = false;
993
+
994
+ const smallRemap = useHead256SmallPrefillAttention(current, ctx);
995
+ if (smallRemap) {
996
+ current = smallRemap;
997
+ changed = true;
998
+ }
999
+
1000
+ const streamingRemap = swapPrefillAttention(current, ctx, {
1001
+ from: 'attention_streaming_f16kv.wgsl',
1002
+ to: 'attention_head256_f16kv.wgsl',
1003
+ });
1004
+ if (streamingRemap) {
1005
+ current = streamingRemap;
1006
+ changed = true;
1007
+ }
1008
+
1009
+ return changed ? current : null;
884
1010
  }
885
1011
 
886
1012
  // =============================================================================
887
1013
  // Transform: widenProjectionWeightsToF32
888
1014
  // =============================================================================
889
1015
 
890
- /** @type {ReadonlySet<string>} */
891
1016
  const PROJECTION_MATMUL_FILES = new Set([
892
1017
  'matmul_gemv_subgroup.wgsl',
893
1018
  'matmul_gemv_subgroup_f16a.wgsl',
@@ -897,10 +1022,9 @@ const PROJECTION_MATMUL_FILES = new Set([
897
1022
  'matmul_f16_tiled.wgsl',
898
1023
  ]);
899
1024
 
900
- /**
1025
+ /*
901
1026
  * Known layer projection ops. Only these are widened; lm_head and embed are
902
1027
  * excluded.
903
- * @type {ReadonlySet<string>}
904
1028
  */
905
1029
  const LAYER_PROJECTION_OPS = new Set([
906
1030
  'q_proj', 'k_proj', 'v_proj', 'o_proj',
@@ -920,7 +1044,7 @@ function resolveDensePrefillProjectionKernel(ctx) {
920
1044
  : 'matmul_f16w_f32a.wgsl';
921
1045
  }
922
1046
 
923
- /**
1047
+ /*
924
1048
  * Replace projection matmul kernels with f32 weight variants.
925
1049
  *
926
1050
  * Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
@@ -928,9 +1052,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
928
1052
  *
929
1053
  * Returns null if no applicable projection kernels are found.
930
1054
  *
931
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
932
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
933
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
934
1055
  */
935
1056
  export function widenProjectionWeightsToF32(graph, ctx) {
936
1057
  // Collect kernel keys used by layer projection steps across all phases
@@ -982,7 +1103,7 @@ export function widenProjectionWeightsToF32(graph, ctx) {
982
1103
  // Transform: remapDenseQ4KPrefillToQ4Native
983
1104
  // =============================================================================
984
1105
 
985
- /**
1106
+ /*
986
1107
  * Replace dense prefill projection kernels with Q4-native prefill variants.
987
1108
  *
988
1109
  * This applies only when the graph already exposes a compatible fused Q4 decode
@@ -993,9 +1114,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
993
1114
  * Returns null when the graph does not have the required dense-prefill + Q4
994
1115
  * decode shape.
995
1116
  *
996
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
997
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
998
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
999
1117
  */
1000
1118
  export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1001
1119
  const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
@@ -1064,7 +1182,7 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1064
1182
  // Transform: remapQ4KPrefillToDense
1065
1183
  // =============================================================================
1066
1184
 
1067
- /**
1185
+ /*
1068
1186
  * Replace fused Q4K prefill projection kernels with dense tiled variants.
1069
1187
  *
1070
1188
  * Decode remains unchanged so the runtime can keep using fused Q4K decode while
@@ -1072,9 +1190,6 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1072
1190
  *
1073
1191
  * Returns null when the graph has no fused Q4K prefill projection kernels.
1074
1192
  *
1075
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1076
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1077
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1078
1193
  */
1079
1194
  export function remapQ4KPrefillToDense(graph, ctx) {
1080
1195
  const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
@@ -1128,7 +1243,7 @@ export function remapQ4KPrefillToDense(graph, ctx) {
1128
1243
  // Transform: useLinearDecodeProjectionF16
1129
1244
  // =============================================================================
1130
1245
 
1131
- /**
1246
+ /*
1132
1247
  * Remap the linear-attention q_proj decode step onto the f16-activation fused
1133
1248
  * Q4 kernel for linear-attention layers only. Full-attention layers keep the
1134
1249
  * manifest-wide f32 activation contract.
@@ -1140,9 +1255,6 @@ export function remapQ4KPrefillToDense(graph, ctx) {
1140
1255
  * repetitive output under greedy decode). q_proj f16 is safe because the
1141
1256
  * linear attention core absorbs the f16 input into its f32 internal state.
1142
1257
  *
1143
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1144
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1145
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1146
1258
  */
1147
1259
  export function useLinearDecodeProjectionF16(graph, ctx) {
1148
1260
  const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
@@ -1198,7 +1310,7 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
1198
1310
  // Transform: remapQ4KDecodeToGemv
1199
1311
  // =============================================================================
1200
1312
 
1201
- /**
1313
+ /*
1202
1314
  * Replace fused Q4K decode projection kernels with GEMV subgroup variants.
1203
1315
  *
1204
1316
  * When Q4K weights have f16 materializations (mixed/dense loader mode), the
@@ -1212,9 +1324,6 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
1212
1324
  * Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
1213
1325
  * attention, residual, activation) are left untouched.
1214
1326
  *
1215
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1216
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1217
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1218
1327
  */
1219
1328
  export function remapQ4KDecodeToGemv(graph, ctx) {
1220
1329
  if (ctx.activationDtype === 'f16') {
@@ -1259,7 +1368,7 @@ export function remapQ4KDecodeToGemv(graph, ctx) {
1259
1368
 
1260
1369
  const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
1261
1370
 
1262
- /**
1371
+ /*
1263
1372
  * Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
1264
1373
  * subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
1265
1374
  *
@@ -1268,9 +1377,6 @@ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj'
1268
1377
  * their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
1269
1378
  * loader remains in mixed-materialization mode.
1270
1379
  *
1271
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1272
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1273
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1274
1380
  */
1275
1381
  export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1276
1382
  if (ctx.activationDtype === 'f16') {
@@ -1327,7 +1433,7 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1327
1433
  // Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
1328
1434
  // =============================================================================
1329
1435
 
1330
- /**
1436
+ /*
1331
1437
  * Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
1332
1438
  * optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
1333
1439
  * cooperative loading with fast nibble extraction for maximum M=1 throughput
@@ -1344,9 +1450,6 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1344
1450
  * isKernelPathFusedQ4K stays true and the weight loader remains in
1345
1451
  * mixed-materialization mode (Q4K retained for attention, f16 for FFN).
1346
1452
  *
1347
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1348
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1349
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1350
1453
  */
1351
1454
  export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
1352
1455
  if (ctx.activationDtype === 'f16') {
@@ -1405,7 +1508,7 @@ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
1405
1508
 
1406
1509
  const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
1407
1510
 
1408
- /**
1511
+ /*
1409
1512
  * Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
1410
1513
  * variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
1411
1514
  *
@@ -1415,9 +1518,6 @@ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
1415
1518
  * fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
1416
1519
  * remains in mixed-materialization mode.
1417
1520
  *
1418
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1419
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1420
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1421
1521
  */
1422
1522
  export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1423
1523
  if (ctx.activationDtype === 'f16') {
@@ -1474,7 +1574,7 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1474
1574
  // Transform: useQwenDecodeF16Matmuls
1475
1575
  // =============================================================================
1476
1576
 
1477
- /**
1577
+ /*
1478
1578
  * Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
1479
1579
  * kernels while keeping the manifest-wide f32 activation contract intact.
1480
1580
  *
@@ -1486,16 +1586,8 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1486
1586
  * FFN down remains on the f32-output contract so the layer residual path stays
1487
1587
  * numerically aligned with the manifest-owned activation dtype.
1488
1588
  *
1489
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1490
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1491
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1492
1589
  */
1493
1590
  export function useQwenDecodeF16Matmuls(graph, ctx) {
1494
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1495
- if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
1496
- return null;
1497
- }
1498
-
1499
1591
  const result = cloneGraph(graph);
1500
1592
  let changed = false;
1501
1593
 
@@ -1560,26 +1652,20 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
1560
1652
  // Transform: useQwenF16PrimaryMatmuls
1561
1653
  // =============================================================================
1562
1654
 
1563
- /**
1564
- * Promote the Qwen 3.5 0.8B execution graph onto its fast f16 primary lane
1565
- * when the runtime explicitly requests f16 activations.
1655
+ /*
1656
+ * Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
1657
+ * lane when the runtime explicitly requests f16 activations.
1566
1658
  *
1567
- * This transform narrows the hot matmul path while keeping `o_proj` on the
1568
- * stable manifest-owned f32-output kernel. The residual stream feeds directly
1569
- * into the next RMSNorm, and the Qwen 3.5 0.8B promoted f16 lane becomes
1570
- * numerically unstable when `o_proj` writes f16 there (observed first at the
1571
- * first full-attention block's post-attention RMSNorm).
1659
+ * This transform narrows the decode projection and LM-head path while keeping
1660
+ * `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
1661
+ * already authored in the manifest on WideTile/head256 kernels and remains
1662
+ * manifest-owned. The residual stream feeds directly into the next RMSNorm, and
1663
+ * the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
1664
+ * `o_proj` writes f16 there (observed first at the first full-attention block's
1665
+ * post-attention RMSNorm).
1572
1666
  *
1573
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1574
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1575
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1576
1667
  */
1577
1668
  export function useQwenF16PrimaryMatmuls(graph, ctx) {
1578
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1579
- if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
1580
- return null;
1581
- }
1582
-
1583
1669
  const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
1584
1670
  if (!layerTypes || layerTypes.length === 0) {
1585
1671
  return null;
@@ -1672,18 +1758,560 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
1672
1758
  return changed ? result : null;
1673
1759
  }
1674
1760
 
1761
+ // =============================================================================
1762
+ // Transform: useQwen36F16Activations
1763
+ // =============================================================================
1764
+
1765
+ /*
1766
+ * Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
1767
+ *
1768
+ */
1769
+ export function useQwen36F16Activations(graph, ctx) {
1770
+ const narrowed = narrowToF16Activations(graph, ctx);
1771
+ const result = narrowed ?? cloneGraph(graph);
1772
+ let changed = narrowed != null;
1773
+
1774
+ const replaceKernelEntry = (key, entry) => {
1775
+ if (!key || !entry) {
1776
+ return;
1777
+ }
1778
+ result.kernels[key] = entry;
1779
+ changed = true;
1780
+ };
1781
+ const replaceOps = (phaseName, ops, kernelKey) => {
1782
+ for (const op of ops) {
1783
+ const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
1784
+ if (phaseResult.changed) {
1785
+ result[phaseName] = phaseResult.steps;
1786
+ changed = true;
1787
+ }
1788
+ }
1789
+ };
1790
+
1791
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
1792
+ const embedKey = embedStep?.[1] ?? null;
1793
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
1794
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
1795
+ replaceKernelEntry(
1796
+ embedKey,
1797
+ deriveKernelEntryWithPrecision(
1798
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
1799
+ { inputDtype: 'f16', outputDtype: 'f16' }
1800
+ )
1801
+ );
1802
+ }
1803
+
1804
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
1805
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
1806
+ const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
1807
+ if (decodeProjectionEntry) {
1808
+ const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
1809
+ result.kernels[f16Key] = decodeProjectionEntry;
1810
+ replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1811
+ }
1812
+
1813
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
1814
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
1815
+ const prefillProjectionEntry =
1816
+ deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
1817
+ ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
1818
+ ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
1819
+ if (prefillProjectionEntry) {
1820
+ const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
1821
+ result.kernels[f16Key] = prefillProjectionEntry;
1822
+ replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1823
+ }
1824
+
1825
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
1826
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
1827
+ replaceKernelEntry(
1828
+ lmHeadKey,
1829
+ deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
1830
+ );
1831
+
1832
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
1833
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
1834
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
1835
+ const lmHeadPrefillF16Entry =
1836
+ deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
1837
+ ?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
1838
+ ?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
1839
+ if (lmHeadPrefillF16Entry) {
1840
+ replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
1841
+ } else if (
1842
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
1843
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
1844
+ ) {
1845
+ replaceKernelEntry(
1846
+ lmHeadPrefillKey,
1847
+ deriveKernelEntryWithPrecision(
1848
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
1849
+ { inputDtype: 'f16', outputDtype: 'f16' }
1850
+ )
1851
+ );
1852
+ }
1853
+
1854
+ return changed ? result : null;
1855
+ }
1856
+
1857
+ // =============================================================================
1858
+ // Transform: useGemma4Int4PleSelectiveF16Decode
1859
+ // =============================================================================
1860
+
1861
+ /*
1862
+ * Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
1863
+ * explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
1864
+ * attention because the repository does not currently have pure-f16 head256 or
1865
+ * head512 prefill kernels.
1866
+ *
1867
+ */
1868
+ export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
1869
+ const result = cloneGraph(graph);
1870
+ let changed = false;
1871
+
1872
+ const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1873
+ const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
1874
+ const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
1875
+ if (decodeProjectionStep && decodeProjectionEntry) {
1876
+ const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
1877
+ result.kernels[decodeProjectionKey] = decodeProjectionEntry;
1878
+ for (const op of ['q_proj', 'k_proj', 'v_proj']) {
1879
+ const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
1880
+ if (phaseResult.changed) {
1881
+ result.decode = phaseResult.steps;
1882
+ changed = true;
1883
+ }
1884
+ }
1885
+ }
1886
+
1887
+ for (const op of ['rope_q', 'rope_k']) {
1888
+ const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
1889
+ const kernelKey = step?.[1];
1890
+ const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1891
+ const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
1892
+ if (!step || !kernelKey || !replacement) {
1893
+ continue;
1894
+ }
1895
+ const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
1896
+ result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
1897
+ deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
1898
+ { inputDtype: 'f16', outputDtype: 'f16' }
1899
+ );
1900
+ const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
1901
+ if (phaseResult.changed) {
1902
+ result.decode = phaseResult.steps;
1903
+ changed = true;
1904
+ }
1905
+ }
1906
+
1907
+ const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
1908
+ const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
1909
+ const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
1910
+ if (attentionStep && attentionEntry) {
1911
+ const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
1912
+ result.kernels[attentionKey] = attentionEntry;
1913
+ const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
1914
+ if (phaseResult.changed) {
1915
+ result.decode = phaseResult.steps;
1916
+ changed = true;
1917
+ }
1918
+ }
1919
+
1920
+ const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
1921
+ const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
1922
+ if (oProjStep && oProjKernel) {
1923
+ const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
1924
+ result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
1925
+ inputDtype: 'f32',
1926
+ outputDtype: 'f32',
1927
+ });
1928
+ const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
1929
+ if (phaseResult.changed) {
1930
+ result.decode = phaseResult.steps;
1931
+ changed = true;
1932
+ }
1933
+ }
1934
+
1935
+ return changed ? result : null;
1936
+ }
1937
+
1938
+ // =============================================================================
1939
+ // Transform: useGemma4TextF16Activations
1940
+ // =============================================================================
1941
+
1942
+ const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
1943
+ 'q_proj',
1944
+ 'k_proj',
1945
+ 'v_proj',
1946
+ 'gate_proj',
1947
+ 'up_proj',
1948
+ ]);
1949
+
1950
+ function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
1951
+ if (!Array.isArray(entries) || !f16Key || !f32Key) {
1952
+ return { entries, changed: false };
1953
+ }
1954
+
1955
+ let changed = false;
1956
+ const remapStep = (step) => {
1957
+ if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
1958
+ return step;
1959
+ }
1960
+ const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
1961
+ ? f32Key
1962
+ : f16Key;
1963
+ if (step[1] === targetKey) {
1964
+ return step;
1965
+ }
1966
+ const replacement = [...step];
1967
+ replacement[1] = targetKey;
1968
+ changed = true;
1969
+ return replacement;
1970
+ };
1971
+
1972
+ const nextEntries = entries.map((entry) => {
1973
+ if (Array.isArray(entry)) {
1974
+ return remapStep(entry);
1975
+ }
1976
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
1977
+ return entry;
1978
+ }
1979
+ return {
1980
+ ...entry,
1981
+ steps: entry.steps.map((step) => remapStep(step)),
1982
+ };
1983
+ });
1984
+
1985
+ return { entries: nextEntries, changed };
1986
+ }
1987
+
1988
+ function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
1989
+ let changed = false;
1990
+
1991
+ const replaceStepWithSourceEntry = (phaseName, op, precision) => {
1992
+ const phase = result[phaseName];
1993
+ const sourcePhase = sourceGraph[phaseName];
1994
+ const step = findPhaseStep(phase, op);
1995
+ const sourceStep = findPhaseStep(sourcePhase, op);
1996
+ const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
1997
+ const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
1998
+ if (!step || !sourceKey || !sourceEntry) {
1999
+ return;
2000
+ }
2001
+ const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
2002
+ result.kernels[stableKey] = precision
2003
+ ? deriveKernelEntryWithPrecision(sourceEntry, precision)
2004
+ : { ...sourceEntry };
2005
+ const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
2006
+ if (phaseResult.changed) {
2007
+ result[phaseName] = phaseResult.steps;
2008
+ changed = true;
2009
+ }
2010
+ };
2011
+
2012
+ for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
2013
+ replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
2014
+ }
2015
+ replaceStepWithSourceEntry('decode', 'attention', {
2016
+ activationDtype: 'f32',
2017
+ kvDtype: 'f16',
2018
+ outputDtype: 'f32',
2019
+ });
2020
+ for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
2021
+ replaceStepWithSourceEntry('postLayer', op, null);
2022
+ }
2023
+
2024
+ return changed;
2025
+ }
2026
+
2027
+ function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
2028
+ const narrowed = narrowToF16Activations(graph, ctx);
2029
+ const result = narrowed ?? cloneGraph(graph);
2030
+ let changed = narrowed != null;
2031
+
2032
+ const replaceKernelEntry = (key, entry) => {
2033
+ if (!key || !entry) {
2034
+ return;
2035
+ }
2036
+ result.kernels[key] = entry;
2037
+ changed = true;
2038
+ };
2039
+ const stableTextBoundary = options?.stableTextBoundary === true;
2040
+
2041
+ const derivePrefillAttentionEntry = (entry) => {
2042
+ if (
2043
+ stableTextBoundary
2044
+ && typeof entry?.kernel === 'string'
2045
+ && entry.kernel === 'attention_small_f16.wgsl'
2046
+ ) {
2047
+ return deriveKernelEntryWithPrecision(
2048
+ deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
2049
+ { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2050
+ );
2051
+ }
2052
+ if (
2053
+ stableTextBoundary
2054
+ && typeof entry?.kernel === 'string'
2055
+ && entry.kernel.endsWith('_f16kv.wgsl')
2056
+ ) {
2057
+ return deriveKernelEntryWithPrecision(
2058
+ entry,
2059
+ { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2060
+ );
2061
+ }
2062
+ return deriveF16AttentionKernelEntry(entry);
2063
+ };
2064
+
2065
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
2066
+ const embedKey = embedStep?.[1] ?? null;
2067
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
2068
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
2069
+ replaceKernelEntry(
2070
+ embedKey,
2071
+ deriveKernelEntryWithPrecision(
2072
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2073
+ { inputDtype: 'f16', outputDtype: 'f16' }
2074
+ )
2075
+ );
2076
+ }
2077
+
2078
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2079
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2080
+ replaceKernelEntry(
2081
+ decodeProjectionKey,
2082
+ deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2083
+ );
2084
+
2085
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2086
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2087
+ const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
2088
+ const prefillProjectionF16Entry = stableTextBoundary
2089
+ ? (
2090
+ deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2091
+ ?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2092
+ ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2093
+ )
2094
+ : (
2095
+ deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2096
+ ?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2097
+ ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2098
+ );
2099
+ if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
2100
+ const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
2101
+ result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
2102
+ changed = true;
2103
+ const remapped = remapGemma412BPrefillProjectionEntries(
2104
+ result.prefill,
2105
+ prefillProjectionF16Key,
2106
+ prefillProjectionKey
2107
+ );
2108
+ result.prefill = remapped.entries;
2109
+ changed = changed || remapped.changed;
2110
+ } else {
2111
+ replaceKernelEntry(
2112
+ prefillProjectionKey,
2113
+ prefillProjectionF16Entry
2114
+ );
2115
+ }
2116
+
2117
+ const replacePrefillAttentionEntries = (entries) => {
2118
+ for (const entry of entries || []) {
2119
+ if (Array.isArray(entry)) {
2120
+ if (entry[0] !== 'attention') {
2121
+ continue;
2122
+ }
2123
+ replaceKernelEntry(
2124
+ entry[1],
2125
+ derivePrefillAttentionEntry(result.kernels[entry[1]])
2126
+ );
2127
+ continue;
2128
+ }
2129
+ if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2130
+ replacePrefillAttentionEntries(entry.steps);
2131
+ }
2132
+ }
2133
+ };
2134
+ replacePrefillAttentionEntries(result.prefill);
2135
+
2136
+ const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2137
+ const finalNormKey = finalNormStep?.[1] ?? null;
2138
+ const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2139
+ if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2140
+ replaceKernelEntry(
2141
+ finalNormKey,
2142
+ deriveKernelEntryWithPrecision(
2143
+ deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2144
+ { inputDtype: 'f16', outputDtype: 'f16' }
2145
+ )
2146
+ );
2147
+ }
2148
+
2149
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2150
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
2151
+ replaceKernelEntry(
2152
+ lmHeadKey,
2153
+ deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2154
+ );
2155
+
2156
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2157
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2158
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2159
+ if (
2160
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2161
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2162
+ ) {
2163
+ replaceKernelEntry(
2164
+ lmHeadPrefillKey,
2165
+ deriveKernelEntryWithPrecision(
2166
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2167
+ { inputDtype: 'f16', outputDtype: 'f16' }
2168
+ )
2169
+ );
2170
+ }
2171
+
2172
+ if (stableTextBoundary) {
2173
+ changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
2174
+ }
2175
+
2176
+ return changed ? result : null;
2177
+ }
2178
+
2179
+ export function useGemma4TextF16Activations(graph, ctx) {
2180
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
2181
+ }
2182
+
2183
+ export function useGemma412BTextF16Activations(graph, ctx) {
2184
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
2185
+ }
2186
+
2187
+ export function useGemma431BTextF16Activations(graph, ctx) {
2188
+ return useGemma4TextF16Activations(graph, ctx);
2189
+ }
2190
+
2191
+ // =============================================================================
2192
+ // Transform: useGemma4Int4PleAf16Activations
2193
+ // =============================================================================
2194
+
2195
+ /*
2196
+ * Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
2197
+ * weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
2198
+ * useGemma4TextF16Activations: same Q4 weight pack, kernels narrowed to f16
2199
+ * activations, prefill projections promoted from widetile to widetile_f16a,
2200
+ * decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
2201
+ * counterparts. Apple Metal stays disabled at the capability layer because the
2202
+ * fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
2203
+ *
2204
+ */
2205
+ export function useGemma4Int4PleAf16Activations(graph, ctx) {
2206
+ const narrowed = narrowToF16Activations(graph, ctx);
2207
+ const result = narrowed ?? cloneGraph(graph);
2208
+ let changed = narrowed != null;
2209
+
2210
+ const replaceKernelEntry = (key, entry) => {
2211
+ if (!key || !entry) {
2212
+ return;
2213
+ }
2214
+ result.kernels[key] = entry;
2215
+ changed = true;
2216
+ };
2217
+
2218
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
2219
+ const embedKey = embedStep?.[1] ?? null;
2220
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
2221
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
2222
+ replaceKernelEntry(
2223
+ embedKey,
2224
+ deriveKernelEntryWithPrecision(
2225
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2226
+ { inputDtype: 'f16', outputDtype: 'f16' }
2227
+ )
2228
+ );
2229
+ }
2230
+
2231
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2232
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2233
+ replaceKernelEntry(
2234
+ decodeProjectionKey,
2235
+ deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2236
+ );
2237
+
2238
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2239
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2240
+ replaceKernelEntry(
2241
+ prefillProjectionKey,
2242
+ deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2243
+ ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
2244
+ ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2245
+ );
2246
+
2247
+ const replacePrefillAttentionEntries = (entries) => {
2248
+ for (const entry of entries || []) {
2249
+ if (Array.isArray(entry)) {
2250
+ if (entry[0] !== 'attention') {
2251
+ continue;
2252
+ }
2253
+ replaceKernelEntry(
2254
+ entry[1],
2255
+ deriveF16AttentionKernelEntry(result.kernels[entry[1]])
2256
+ );
2257
+ continue;
2258
+ }
2259
+ if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2260
+ replacePrefillAttentionEntries(entry.steps);
2261
+ }
2262
+ }
2263
+ };
2264
+ replacePrefillAttentionEntries(result.prefill);
2265
+
2266
+ const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2267
+ const finalNormKey = finalNormStep?.[1] ?? null;
2268
+ const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2269
+ if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2270
+ replaceKernelEntry(
2271
+ finalNormKey,
2272
+ deriveKernelEntryWithPrecision(
2273
+ deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2274
+ { inputDtype: 'f16', outputDtype: 'f16' }
2275
+ )
2276
+ );
2277
+ }
2278
+
2279
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2280
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
2281
+ replaceKernelEntry(
2282
+ lmHeadKey,
2283
+ deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2284
+ );
2285
+
2286
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2287
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2288
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2289
+ if (
2290
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2291
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2292
+ ) {
2293
+ replaceKernelEntry(
2294
+ lmHeadPrefillKey,
2295
+ deriveKernelEntryWithPrecision(
2296
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2297
+ { inputDtype: 'f16', outputDtype: 'f16' }
2298
+ )
2299
+ );
2300
+ }
2301
+
2302
+ return changed ? result : null;
2303
+ }
2304
+
1675
2305
  // =============================================================================
1676
2306
  // Composition
1677
2307
  // =============================================================================
1678
2308
 
1679
- /**
2309
+ /*
1680
2310
  * Compose multiple transforms into a single transform function.
1681
2311
  *
1682
2312
  * Each transform is applied sequentially. If a transform returns null
1683
2313
  * (not applicable), the graph passes through unchanged.
1684
2314
  *
1685
- * @param {Array<(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph | null>} transforms
1686
- * @returns {(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph}
1687
2315
  */
1688
2316
  export function composeTransforms(...transforms) {
1689
2317
  return (graph, ctx) => {
@@ -1698,17 +2326,42 @@ export function composeTransforms(...transforms) {
1698
2326
  };
1699
2327
  }
1700
2328
 
2329
+ // Session-only capability transform marker. The execution-v1 compiler applies
2330
+ // the matching runtime session patch; the graph itself is intentionally stable.
2331
+ export function disableRetainQ4KMaterialization() {
2332
+ return null;
2333
+ }
2334
+
2335
+ /*
2336
+ * Fail-closed sentinel transform. A capability rule installs this when the
2337
+ * matched (modelId, runtime profile) combination is contradictory — for
2338
+ * example, an af32 manifest variant paired with a runtime profile that
2339
+ * demands f16 activations. The matcher reaches this transform only when the
2340
+ * earlier manifest-binding gate has been bypassed; throwing here keeps the
2341
+ * lane-confusion door shut at the capability layer too.
2342
+ *
2343
+ */
2344
+ export function failClosedLaneMismatch(_graph, ctx) {
2345
+ const modelId = ctx?.modelId ?? 'unknown';
2346
+ const activationDtype = ctx?.activationDtype ?? 'unknown';
2347
+ throw new Error(
2348
+ `Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
2349
+ 'The manifest variant tag is the lane identity — load the manifest variant ' +
2350
+ 'whose compute lane matches the runtime profile.'
2351
+ );
2352
+ }
2353
+
1701
2354
  // =============================================================================
1702
2355
  // Registry
1703
2356
  // =============================================================================
1704
2357
 
1705
- /** @type {Readonly<Record<string, Function>>} */
1706
2358
  export const TRANSFORMS = Object.freeze({
1707
2359
  narrowToF16Activations,
1708
2360
  removeSubgroups,
1709
2361
  widenToF32Activations,
1710
2362
  widenToF32CorrectnessFallback,
1711
2363
  swapPrefillAttention,
2364
+ useHead256SmallPrefillAttention,
1712
2365
  useHead256PrefillAttention,
1713
2366
  widenProjectionWeightsToF32,
1714
2367
  remapDenseQ4KPrefillToQ4Native,
@@ -1718,7 +2371,15 @@ export const TRANSFORMS = Object.freeze({
1718
2371
  remapQ4KDecodeAttentionToGemv,
1719
2372
  remapQ4KDecodeAttentionToFusedQ4KGemv,
1720
2373
  remapQ4KDecodeFFNToGemv,
2374
+ disableRetainQ4KMaterialization,
1721
2375
  useQwenF16PrimaryMatmuls,
2376
+ useQwen36F16Activations,
1722
2377
  useQwenDecodeF16Matmuls,
2378
+ useGemma4Int4PleSelectiveF16Decode,
2379
+ useGemma4TextF16Activations,
2380
+ useGemma412BTextF16Activations,
2381
+ useGemma431BTextF16Activations,
2382
+ useGemma4Int4PleAf16Activations,
2383
+ failClosedLaneMismatch,
1723
2384
  composeTransforms,
1724
2385
  });