doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -1,4 +1,10 @@
1
- // =============================================================================
1
+
2
+ import { F16_TO_F32_ACTIVATION_MAP, F32_TO_F16_ACTIVATION_MAP, KERNEL_FILE_PRECISION_PATCHES, cloneGraph, deriveF16AttentionKernelEntry, deriveKernelEntry, deriveKernelEntryWithPrecision, deriveKernelKey, deriveLinearDecodeF16KernelEntry, deriveLmHeadDecodeF16KernelEntry, deriveQ4DecodeF16KernelEntry, deriveQ4PrefillF16AccumKernelEntry, deriveQ4PrefillF16KernelEntry, deriveQ4WideTilePrefillF16KernelEntry, findPhaseStep, narrowToF16Activations, replacePhaseStepKernelKey, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';
3
+ import { LAYER_PROJECTION_OPS, useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode, useGemma4TextF16ActivationsForLane } from './execution-graph/fusion.js';
4
+ import { deriveMappedKernelEntry, deriveQ4DecodeF32ActivationKernelEntry, deriveQ4PrefillF32ActivationKernelEntry, findKernelKeyByFileAndEntry, hasKernelFile, remapDenseQ4KPrefillToQ4Native, remapStepKeys, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
5
+ export { remapDenseQ4KPrefillToQ4Native, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
6
+ export { useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode } from './execution-graph/fusion.js';
7
+ export { narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';// =============================================================================
2
8
  // Execution Graph Transforms
3
9
  // =============================================================================
4
10
  //
@@ -16,38 +22,11 @@
16
22
  /*
17
23
  * Deep-clone an execution graph.
18
24
  */
19
- function cloneGraph(graph) {
20
- return structuredClone(graph);
21
- }
22
25
 
23
26
  /*
24
27
  * Shader files that require subgroups even though "subgroup" is not in the filename.
25
28
  * Online attention kernels use subgroup reductions internally.
26
29
  */
27
- const SUBGROUP_REQUIRING_FILES = new Set([
28
- 'attention_decode_online_f16kv.wgsl',
29
- 'attention_decode_online_f16.wgsl',
30
- ]);
31
-
32
- const KERNEL_FILE_PRECISION_PATCHES = new Map([
33
- ['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
34
- ['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
35
- ['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
36
- ['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
37
- ['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
38
- ['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
39
- ['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
40
- ['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
41
- ['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
42
- ['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
43
- ['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
44
- ['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
45
- ['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
46
- ['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
47
- ['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
48
- ['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
49
- ['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
50
- ]);
51
30
 
52
31
  export function getKernelFilePrecisionPatch(kernel) {
53
32
  return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
@@ -56,114 +35,26 @@ export function getKernelFilePrecisionPatch(kernel) {
56
35
  /*
57
36
  * Check whether a kernel entry requires subgroup support.
58
37
  */
59
- function isSubgroupKernel(kernelEntry) {
60
- if (typeof kernelEntry.kernel !== 'string') return false;
61
- return kernelEntry.kernel.includes('subgroup') || SUBGROUP_REQUIRING_FILES.has(kernelEntry.kernel);
62
- }
63
-
64
- function requiresNoSubgroupFallback(kernelEntry) {
65
- if (typeof kernelEntry?.kernel !== 'string') return false;
66
- return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
67
- }
68
38
 
69
39
  /*
70
40
  * Find all kernel keys in the graph whose `kernel` file matches the given filename.
71
41
  */
72
- function findKernelKeysByFile(graph, filename) {
73
- const keys = [];
74
- for (const [key, entry] of Object.entries(graph.kernels)) {
75
- if (entry.kernel === filename) {
76
- keys.push(key);
77
- }
78
- }
79
- return keys;
80
- }
81
42
 
82
43
  /*
83
44
  * Check whether any kernel in the graph uses the given shader file.
84
45
  */
85
- function hasKernelFile(graph, filename) {
86
- return findKernelKeysByFile(graph, filename).length > 0;
87
- }
88
46
 
89
47
  /*
90
48
  * Create a new kernel entry with the digest cleared (shader changed).
91
49
  */
92
- function deriveKernelEntry(base, newFile, newEntry, constants) {
93
- const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
94
- if (constants === null) {
95
- delete derived.constants;
96
- } else if (constants !== undefined) {
97
- derived.constants = { ...constants };
98
- }
99
- const precision = deriveKernelPrecision(base, newFile);
100
- if (precision) {
101
- derived.precision = precision;
102
- } else {
103
- delete derived.precision;
104
- }
105
- return derived;
106
- }
107
-
108
- function deriveMappedKernelEntry(base, newFile) {
109
- const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
110
- return deriveKernelEntry(base, newFile, newEntry);
111
- }
112
-
113
- function deriveKernelPrecision(base, newFile) {
114
- const precision = base.precision ? { ...base.precision } : {};
115
- const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
116
- if (precisionPatch) {
117
- Object.assign(precision, precisionPatch);
118
- }
119
- if (!String(newFile).startsWith('attention')) {
120
- return Object.keys(precision).length > 0 ? precision : null;
121
- }
122
- if (newFile.includes('_f16kv')) {
123
- precision.activationDtype = 'f32';
124
- precision.kvDtype = 'f16';
125
- return precision;
126
- }
127
- if (newFile.includes('_f16')) {
128
- precision.activationDtype = 'f16';
129
- precision.kvDtype = 'f16';
130
- return precision;
131
- }
132
- precision.activationDtype = 'f32';
133
- precision.kvDtype = 'f32';
134
- return precision;
135
- }
136
50
 
137
51
  /*
138
52
  * Derive a non-colliding kernel key name.
139
53
  */
140
- function deriveKernelKey(kernels, baseKey, suffix) {
141
- const candidate = `${baseKey}${suffix}`;
142
- if (!kernels[candidate]) {
143
- return candidate;
144
- }
145
- let counter = 2;
146
- while (kernels[`${candidate}_${counter}`]) {
147
- counter++;
148
- }
149
- return `${candidate}_${counter}`;
150
- }
151
54
 
152
55
  /*
153
56
  * Replace kernel key references in step tuples.
154
57
  */
155
- function remapStepKeys(steps, keyMap) {
156
- return steps.map((step) => {
157
- const kernelKey = step[1];
158
- const replacement = keyMap.get(kernelKey);
159
- if (replacement !== undefined) {
160
- const newStep = [...step];
161
- newStep[1] = replacement;
162
- return newStep;
163
- }
164
- return step;
165
- });
166
- }
167
58
 
168
59
  /*
169
60
  * Check whether a step tuple's kernel key resolves to the given shader file.
@@ -178,30 +69,10 @@ function stepUsesFile(graph, step, filename) {
178
69
  * Find the first kernel key used by matching ops in a phase whose shader file
179
70
  * satisfies the provided predicate.
180
71
  */
181
- function findPhaseKernelKey(graph, steps, ops, predicate) {
182
- for (const step of steps || []) {
183
- if (!ops.has(step[0])) {
184
- continue;
185
- }
186
- const entry = graph.kernels[step[1]];
187
- if (entry && predicate(entry)) {
188
- return step[1];
189
- }
190
- }
191
- return null;
192
- }
193
72
 
194
73
  /*
195
74
  * Find an existing kernel key by shader file and entry point.
196
75
  */
197
- function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
198
- for (const [key, entry] of Object.entries(graph.kernels)) {
199
- if (entry.kernel === filename && entry.entry === entryPoint) {
200
- return key;
201
- }
202
- }
203
- return null;
204
- }
205
76
 
206
77
  function normalizeLayerType(layerType) {
207
78
  return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
@@ -251,306 +122,6 @@ function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey)
251
122
  return groupedEntries;
252
123
  }
253
124
 
254
- function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
255
- if (!Array.isArray(steps) || steps.length === 0 || !replacementKernelKey) {
256
- return { steps, changed: false };
257
- }
258
- let changed = false;
259
- const nextSteps = steps.map((step) => {
260
- if (!Array.isArray(step) || step[0] !== op) {
261
- return step;
262
- }
263
- if (step[1] === replacementKernelKey) {
264
- return step;
265
- }
266
- const replacement = [...step];
267
- replacement[1] = replacementKernelKey;
268
- changed = true;
269
- return replacement;
270
- });
271
- return { steps: nextSteps, changed };
272
- }
273
-
274
- function findPhaseStep(steps, op) {
275
- if (!Array.isArray(steps) || !op) {
276
- return null;
277
- }
278
- for (const entry of steps) {
279
- if (Array.isArray(entry)) {
280
- if (entry[0] === op) {
281
- return entry;
282
- }
283
- continue;
284
- }
285
- if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
286
- continue;
287
- }
288
- const nested = findPhaseStep(entry.steps, op);
289
- if (nested) {
290
- return nested;
291
- }
292
- }
293
- return null;
294
- }
295
-
296
- function deriveKernelEntryWithPrecision(base, precision) {
297
- return {
298
- ...base,
299
- precision: {
300
- ...(base.precision ?? {}),
301
- ...precision,
302
- },
303
- };
304
- }
305
-
306
- const ATTENTION_F16KV_TO_F16_MAP = new Map([
307
- ['attention_decode_online_f16kv.wgsl', 'attention_decode_online_f16.wgsl'],
308
- ['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
309
- ['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
310
- ['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
311
- ['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
312
- ]);
313
-
314
- function deriveF16AttentionKernelEntry(base) {
315
- if (typeof base?.kernel !== 'string') {
316
- return null;
317
- }
318
- const replacement = ATTENTION_F16KV_TO_F16_MAP.get(base.kernel);
319
- if (!replacement) {
320
- return null;
321
- }
322
- return deriveKernelEntryWithPrecision(
323
- deriveKernelEntry(base, replacement, base.entry),
324
- { activationDtype: 'f16', kvDtype: 'f16' }
325
- );
326
- }
327
-
328
- function deriveLinearDecodeF16KernelEntry(base) {
329
- const precision = {
330
- inputDtype: 'f16',
331
- outputDtype: 'f16',
332
- };
333
- if (base.kernel === 'fused_matmul_q4.wgsl' && base.entry === 'main_multicol') {
334
- return {
335
- ...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a'),
336
- precision: {
337
- ...(base.precision ?? {}),
338
- ...precision,
339
- },
340
- };
341
- }
342
- if (
343
- (base.kernel === 'fused_matmul_q4_multicol_f16.wgsl' && base.entry === 'main_multicol_f16')
344
- || (base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl' && base.entry === 'main_multicol_f16a')
345
- ) {
346
- return deriveKernelEntryWithPrecision(base, precision);
347
- }
348
- return null;
349
- }
350
-
351
- function deriveLmHeadDecodeF16KernelEntry(base) {
352
- const precision = {
353
- inputDtype: 'f16',
354
- outputDtype: 'f16',
355
- };
356
- if (base.kernel === 'matmul_gemv_subgroup.wgsl' && base.entry === 'main_multicol') {
357
- return {
358
- ...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', 'main_multicol'),
359
- precision: {
360
- ...(base.precision ?? {}),
361
- ...precision,
362
- },
363
- };
364
- }
365
- if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl' && base.entry === 'main_multicol') {
366
- return deriveKernelEntryWithPrecision(base, precision);
367
- }
368
- return null;
369
- }
370
-
371
- function deriveDenseDecodeF16KernelEntry(base) {
372
- if (typeof base?.kernel !== 'string') {
373
- return null;
374
- }
375
- const precision = {
376
- inputDtype: 'f16',
377
- outputDtype: 'f16',
378
- };
379
- if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
380
- return {
381
- ...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
382
- precision: {
383
- ...(base.precision ?? {}),
384
- ...precision,
385
- },
386
- };
387
- }
388
- if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
389
- return deriveKernelEntryWithPrecision(base, precision);
390
- }
391
- return null;
392
- }
393
-
394
- function deriveQ4DecodeF16KernelEntry(base) {
395
- if (typeof base?.kernel !== 'string') {
396
- return null;
397
- }
398
- const precision = {
399
- inputDtype: 'f16',
400
- outputDtype: 'f16',
401
- };
402
- if (base.kernel === 'fused_matmul_q4.wgsl') {
403
- return {
404
- ...deriveKernelEntry(base, 'fused_matmul_q4_multicol_f16a.wgsl', 'main_multicol_f16a', null),
405
- precision: {
406
- ...(base.precision ?? {}),
407
- ...precision,
408
- },
409
- };
410
- }
411
- if (
412
- base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
413
- || base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
414
- ) {
415
- return deriveKernelEntryWithPrecision(base, precision);
416
- }
417
- return null;
418
- }
419
-
420
- function deriveQ4PrefillF16KernelEntry(base) {
421
- if (typeof base?.kernel !== 'string') {
422
- return null;
423
- }
424
- const precision = {
425
- inputDtype: 'f16',
426
- outputDtype: 'f16',
427
- };
428
- if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
429
- return deriveKernelEntryWithPrecision(base, precision);
430
- }
431
- if (base.kernel.startsWith('fused_matmul_q4_batched')) {
432
- return {
433
- ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
434
- precision: {
435
- ...(base.precision ?? {}),
436
- ...precision,
437
- },
438
- };
439
- }
440
- return null;
441
- }
442
-
443
- function deriveQ4WideTilePrefillF16KernelEntry(base) {
444
- if (typeof base?.kernel !== 'string') {
445
- return null;
446
- }
447
- const precision = {
448
- inputDtype: 'f16',
449
- outputDtype: 'f16',
450
- };
451
- if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
452
- return {
453
- ...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
454
- precision: {
455
- ...(base.precision ?? {}),
456
- ...precision,
457
- },
458
- };
459
- }
460
- if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
461
- return deriveKernelEntryWithPrecision(base, precision);
462
- }
463
- return null;
464
- }
465
-
466
- function deriveQ4PrefillF16AccumKernelEntry(base) {
467
- if (typeof base?.kernel !== 'string') {
468
- return null;
469
- }
470
- const precision = {
471
- inputDtype: 'f16',
472
- outputDtype: 'f16',
473
- };
474
- if (
475
- base.kernel === 'fused_matmul_q4_widetile.wgsl'
476
- || base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
477
- || base.kernel.startsWith('fused_matmul_q4_batched')
478
- ) {
479
- return {
480
- ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
481
- precision: {
482
- ...(base.precision ?? {}),
483
- ...precision,
484
- },
485
- };
486
- }
487
- return null;
488
- }
489
-
490
- function deriveQ4DecodeF32ActivationKernelEntry(base) {
491
- if (typeof base?.kernel !== 'string') {
492
- return null;
493
- }
494
- const precision = {
495
- inputDtype: 'f32',
496
- outputDtype: 'f32',
497
- };
498
- if (
499
- base.kernel === 'fused_matmul_q4_multicol_f16.wgsl'
500
- || base.kernel === 'fused_matmul_q4_multicol_f16a.wgsl'
501
- ) {
502
- return deriveKernelEntryWithPrecision(
503
- deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main_multicol', null),
504
- precision
505
- );
506
- }
507
- if (base.kernel === 'fused_matmul_q4_f16a.wgsl') {
508
- return deriveKernelEntryWithPrecision(
509
- deriveKernelEntry(base, 'fused_matmul_q4.wgsl', 'main', null),
510
- precision
511
- );
512
- }
513
- return null;
514
- }
515
-
516
- function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
517
- if (typeof base?.kernel !== 'string') {
518
- return null;
519
- }
520
- const replacement = options.fullF32 === true
521
- ? {
522
- kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
523
- entry: 'main',
524
- }
525
- : {
526
- kernel: 'fused_matmul_q4_widetile.wgsl',
527
- entry: 'main',
528
- };
529
- if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
530
- return deriveKernelEntryWithPrecision(
531
- deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
532
- { inputDtype: 'f32', outputDtype: 'f32' }
533
- );
534
- }
535
- if (
536
- base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
537
- && base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
538
- && base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
539
- ) {
540
- return null;
541
- }
542
- if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
543
- return deriveKernelEntryWithPrecision(
544
- deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
545
- { inputDtype: 'f32', outputDtype: 'f32' }
546
- );
547
- }
548
- return deriveKernelEntryWithPrecision(
549
- deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
550
- { inputDtype: 'f32', outputDtype: 'f32' }
551
- );
552
- }
553
-
554
125
  function replacePhaseStepEntries(steps, op, replacementEntries) {
555
126
  if (!Array.isArray(steps) || steps.length === 0 || !Array.isArray(replacementEntries) || replacementEntries.length === 0) {
556
127
  return { steps, changed: false };
@@ -580,86 +151,6 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
580
151
  * Returns null if the graph has no subgroup kernels.
581
152
  *
582
153
  */
583
- export function removeSubgroups(graph, ctx) {
584
- const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
585
- if (!hasAnyFallbackKernel) {
586
- return null;
587
- }
588
-
589
- const result = cloneGraph(graph);
590
- const keyMap = new Map();
591
- const isF16Activation = ctx.activationDtype === 'f16';
592
-
593
- // Build replacement kernel entries for each subgroup or fused-Q4K kernel
594
- // reference found in decode, prefill, and postLayer steps.
595
- const decodeKeys = new Set((result.decode || []).map((s) => s[1]));
596
- const prefillKeys = new Set((result.prefill || []).map((s) => s[1]));
597
- const postLayerKeys = new Set((result.postLayer || []).map((s) => s[1]));
598
- const relevantKeys = new Set([...decodeKeys, ...prefillKeys, ...postLayerKeys]);
599
-
600
- for (const key of relevantKeys) {
601
- const entry = result.kernels[key];
602
- if (!entry || !requiresNoSubgroupFallback(entry)) {
603
- continue;
604
- }
605
-
606
- const isPostLayer = postLayerKeys.has(key) && !decodeKeys.has(key);
607
- const isMulticol = entry.entry === 'main_multicol';
608
- const isLmHead = isPostLayer || isMulticol;
609
-
610
- let newFile;
611
- let newEntry = 'main';
612
- let newConstants = undefined;
613
-
614
- if (entry.kernel === 'matmul_gemv_subgroup.wgsl') {
615
- if (isLmHead) {
616
- // lm_head: multicol → plain matmul, remove MULTICOL constants
617
- newFile = 'matmul_f16w_f32a.wgsl';
618
- newConstants = null;
619
- } else {
620
- // decode projections: vec4 → tiled matmul
621
- newFile = 'matmul_f16w_f32a_tiled.wgsl';
622
- }
623
- } else if (entry.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
624
- if (isLmHead) {
625
- newFile = isF16Activation ? 'matmul_f16.wgsl' : 'matmul_f16w_f32a.wgsl';
626
- newConstants = null;
627
- } else {
628
- newFile = isF16Activation ? 'matmul_f16.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
629
- }
630
- } else if (entry.kernel === 'attention_decode_online_f16kv.wgsl') {
631
- // f16kv online uses f32 Q; if activations are f16, fall back to all-f16 chunked
632
- newFile = isF16Activation
633
- ? 'attention_decode_chunked_f16.wgsl'
634
- : 'attention_decode_chunked_f16kv.wgsl';
635
- newEntry = entry.entry;
636
- } else if (entry.kernel === 'attention_decode_online_f16.wgsl') {
637
- newFile = 'attention_decode_chunked_f16.wgsl';
638
- newEntry = entry.entry;
639
- } else if (entry.kernel.startsWith('fused_matmul_q4')) {
640
- newFile = isF16Activation ? 'matmul_f16_tiled.wgsl' : 'matmul_f16w_f32a_tiled.wgsl';
641
- newConstants = null;
642
- } else {
643
- // Unknown subgroup kernel — skip
644
- continue;
645
- }
646
-
647
- const newKey = deriveKernelKey(result.kernels, key, '_nosg');
648
- result.kernels[newKey] = deriveKernelEntry(entry, newFile, newEntry, newConstants);
649
- keyMap.set(key, newKey);
650
- }
651
-
652
- if (keyMap.size === 0) {
653
- return null;
654
- }
655
-
656
- // Remap decode, prefill, and postLayer steps; leave preLayer untouched
657
- result.decode = remapStepKeys(result.decode || [], keyMap);
658
- result.prefill = remapStepKeys(result.prefill || [], keyMap);
659
- result.postLayer = remapStepKeys(result.postLayer || [], keyMap);
660
-
661
- return result;
662
- }
663
154
 
664
155
  // =============================================================================
665
156
  // Transform: widenToF32Activations
@@ -670,25 +161,6 @@ export function removeSubgroups(graph, ctx) {
670
161
  * that still use f16 for weights and KV cache. Requires shader-f16 for weight
671
162
  * and KV buffer reads.
672
163
  */
673
- const F16_TO_F32_ACTIVATION_MAP = new Map([
674
- ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
675
- ['rope_f16.wgsl', 'rope.wgsl'],
676
- ['residual_f16.wgsl', 'residual.wgsl'],
677
- ['gelu_f16.wgsl', 'gelu.wgsl'],
678
- ['silu_f16.wgsl', 'silu.wgsl'],
679
- ['sample_f16.wgsl', 'sample.wgsl'],
680
- ['gather_f16.wgsl', 'gather.wgsl'],
681
- ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
682
- ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
683
- ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
684
- ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
685
- ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
686
- ['attention_decode_online_f16.wgsl', 'attention_decode_online_f16kv.wgsl'],
687
- ['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
688
- ['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
689
- ['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
690
- ['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
691
- ]);
692
164
 
693
165
  export function resolveF16ToF32ActivationKernel(kernel) {
694
166
  return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
@@ -702,25 +174,13 @@ export function resolveF16ToF32ActivationKernel(kernel) {
702
174
  * runtime session explicitly requests f16 activations for an execution-v1
703
175
  * graph that was authored with conservative f32 activation defaults.
704
176
  */
705
- const F32_TO_F16_ACTIVATION_MAP = new Map(
706
- Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
707
- );
177
+
708
178
  F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
709
179
  F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
710
180
  // head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
711
181
  // f16 requests fail-closed until that path has its own evidence.
712
182
  F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
713
183
 
714
- function hasExplicitF32ActivationContract(entry) {
715
- const precision = entry?.precision;
716
- if (!precision || typeof precision !== 'object') {
717
- return false;
718
- }
719
- return precision.activationDtype === 'f32'
720
- || precision.inputDtype === 'f32'
721
- || precision.outputDtype === 'f32';
722
- }
723
-
724
184
  /*
725
185
  * Correctness fallback: preserve f16 weights where possible, but widen both
726
186
  * activations and KV-cache interactions onto the stable f32 execution lane.
@@ -754,37 +214,6 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
754
214
  * pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
755
215
  * Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
756
216
  */
757
- const FULL_F32_SHADER_MAP = new Map([
758
- // f16-activation utility kernels → f32
759
- ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
760
- ['rope_f16.wgsl', 'rope.wgsl'],
761
- ['residual_f16.wgsl', 'residual.wgsl'],
762
- ['gelu_f16.wgsl', 'gelu.wgsl'],
763
- ['silu_f16.wgsl', 'silu.wgsl'],
764
- ['sample_f16.wgsl', 'sample.wgsl'],
765
- ['gather_f16.wgsl', 'gather.wgsl'],
766
- ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
767
- ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
768
- // f16-activation matmul → f32
769
- ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
770
- ['matmul_f16.wgsl', 'matmul_f32.wgsl'],
771
- ['matmul_f16_tiled.wgsl', 'matmul_f32.wgsl'],
772
- // f16-weight + f32-activation matmul → f32
773
- ['matmul_gemv_subgroup.wgsl', 'matmul_f32.wgsl'],
774
- ['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
775
- ['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
776
- // f16-activation attention → f32
777
- ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
778
- ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
779
- ['attention_small_f16.wgsl', 'attention_small.wgsl'],
780
- ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
781
- // f16kv attention (f32 Q, f16 KV) → f32
782
- ['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
783
- ['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
784
- ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
785
- ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
786
- ['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
787
- ]);
788
217
 
789
218
  /*
790
219
  * Widen all f16-activation shaders to f32-activation equivalents.
@@ -796,46 +225,6 @@ const FULL_F32_SHADER_MAP = new Map([
796
225
  * to reflect the widened dtype.
797
226
  *
798
227
  */
799
- export function widenToF32Activations(graph, ctx) {
800
- // Bail out if fused f16 FFN is present — no direct f32 equivalent
801
- if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
802
- return null;
803
- }
804
-
805
- // When the GPU cannot compile any f16 WGSL (hasF16=false), use the full f32
806
- // map that also covers f16-weight and f16-KV kernels. Otherwise use the
807
- // activation-only map that preserves f16 weights/KV for precision fallback.
808
- const shaderMap = ctx.capabilities?.hasF16 === false || ctx.kvDtype === 'f32'
809
- ? FULL_F32_SHADER_MAP
810
- : F16_TO_F32_ACTIVATION_MAP;
811
- const fullF32 = ctx.capabilities?.hasF16 === false || ctx.kvDtype === 'f32';
812
-
813
- const hasTargetShader = Object.values(graph.kernels).some(
814
- (entry) => shaderMap.has(entry.kernel)
815
- || deriveQ4DecodeF32ActivationKernelEntry(entry)
816
- || deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
817
- );
818
- if (!hasTargetShader) {
819
- return null;
820
- }
821
-
822
- const result = cloneGraph(graph);
823
-
824
- for (const [key, entry] of Object.entries(result.kernels)) {
825
- const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
826
- ?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
827
- if (q4FallbackEntry) {
828
- result.kernels[key] = q4FallbackEntry;
829
- continue;
830
- }
831
- const replacement = shaderMap.get(entry.kernel);
832
- if (replacement !== undefined) {
833
- result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
834
- }
835
- }
836
-
837
- return result;
838
- }
839
228
 
840
229
  /*
841
230
  * Widen an f16 execution graph onto the stable f32 correctness lane used for
@@ -876,32 +265,8 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
876
265
  * Returns null if the graph has no supported f32-activation kernels to swap or
877
266
  * if the runtime did not explicitly request f16 activations on an f16-capable
878
267
  * GPU.
879
- *
880
- */
881
- export function narrowToF16Activations(graph, ctx) {
882
- if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
883
- return null;
884
- }
885
-
886
- const hasTargetShader = Object.values(graph.kernels).some(
887
- (entry) => !hasExplicitF32ActivationContract(entry) && F32_TO_F16_ACTIVATION_MAP.has(entry.kernel)
888
- );
889
- if (!hasTargetShader) {
890
- return null;
891
- }
892
-
893
- const result = cloneGraph(graph);
894
- for (const [key, entry] of Object.entries(result.kernels)) {
895
- if (hasExplicitF32ActivationContract(entry)) {
896
- continue;
897
- }
898
- const replacement = F32_TO_F16_ACTIVATION_MAP.get(entry.kernel);
899
- if (replacement !== undefined) {
900
- result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
901
- }
902
- }
903
- return result;
904
- }
268
+ *
269
+ */
905
270
 
906
271
  // =============================================================================
907
272
  // Transform: swapPrefillAttention
@@ -1021,30 +386,10 @@ export function useHead256PrefillAttention(graph, ctx) {
1021
386
  // Transform: widenProjectionWeightsToF32
1022
387
  // =============================================================================
1023
388
 
1024
- const PROJECTION_MATMUL_FILES = new Set([
1025
- 'matmul_gemv_subgroup.wgsl',
1026
- 'matmul_gemv_subgroup_f16a.wgsl',
1027
- 'matmul_f16w_f32a_tiled.wgsl',
1028
- 'matmul_f16w_f32a.wgsl',
1029
- 'matmul_f16.wgsl',
1030
- 'matmul_f16_tiled.wgsl',
1031
- ]);
1032
-
1033
389
  /*
1034
390
  * Known layer projection ops. Only these are widened; lm_head and embed are
1035
391
  * excluded.
1036
392
  */
1037
- const LAYER_PROJECTION_OPS = new Set([
1038
- 'q_proj', 'k_proj', 'v_proj', 'o_proj',
1039
- 'gate_proj', 'up_proj', 'down_proj',
1040
- ]);
1041
-
1042
- const DENSE_Q4_PREFILL_FILES = new Set([
1043
- 'matmul_f16w_f32a.wgsl',
1044
- 'matmul_f16w_f32a_tiled.wgsl',
1045
- 'matmul_f16.wgsl',
1046
- 'matmul_f16_tiled.wgsl',
1047
- ]);
1048
393
 
1049
394
  function resolveDensePrefillProjectionKernel(ctx) {
1050
395
  return ctx.activationDtype === 'f16'
@@ -1061,51 +406,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
1061
406
  * Returns null if no applicable projection kernels are found.
1062
407
  *
1063
408
  */
1064
- export function widenProjectionWeightsToF32(graph, ctx) {
1065
- // Collect kernel keys used by layer projection steps across all phases
1066
- const projectionKernelKeys = new Set();
1067
- const allPhases = ['preLayer', 'decode', 'prefill', 'postLayer'];
1068
-
1069
- for (const phase of allPhases) {
1070
- const steps = graph[phase];
1071
- if (!Array.isArray(steps)) {
1072
- continue;
1073
- }
1074
- for (const step of steps) {
1075
- const op = step[0];
1076
- const kernelKey = step[1];
1077
- if (LAYER_PROJECTION_OPS.has(op) && kernelKey) {
1078
- projectionKernelKeys.add(kernelKey);
1079
- }
1080
- }
1081
- }
1082
-
1083
- if (projectionKernelKeys.size === 0) {
1084
- return null;
1085
- }
1086
-
1087
- // Check whether any of those keys reference a swappable matmul
1088
- const keysToSwap = new Set();
1089
- for (const key of projectionKernelKeys) {
1090
- const entry = graph.kernels[key];
1091
- if (entry && PROJECTION_MATMUL_FILES.has(entry.kernel)) {
1092
- keysToSwap.add(key);
1093
- }
1094
- }
1095
-
1096
- if (keysToSwap.size === 0) {
1097
- return null;
1098
- }
1099
-
1100
- const result = cloneGraph(graph);
1101
-
1102
- for (const key of keysToSwap) {
1103
- const entry = result.kernels[key];
1104
- result.kernels[key] = deriveKernelEntry(entry, 'matmul_f32.wgsl', 'main');
1105
- }
1106
-
1107
- return result;
1108
- }
1109
409
 
1110
410
  // =============================================================================
1111
411
  // Transform: remapDenseQ4KPrefillToQ4Native
@@ -1123,68 +423,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
1123
423
  * decode shape.
1124
424
  *
1125
425
  */
1126
- export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1127
- const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
1128
- if (!LAYER_PROJECTION_OPS.has(step[0])) {
1129
- return false;
1130
- }
1131
- const entry = graph.kernels[step[1]];
1132
- return entry != null && DENSE_Q4_PREFILL_FILES.has(entry.kernel);
1133
- });
1134
- if (densePrefillProjectionSteps.length === 0) {
1135
- return null;
1136
- }
1137
-
1138
- const result = cloneGraph(graph);
1139
- const existingSharedKey = findKernelKeyByFileAndEntry(
1140
- result,
1141
- 'fused_matmul_q4_batched_multicol_shared.wgsl',
1142
- 'main'
1143
- );
1144
- let sharedKey = existingSharedKey;
1145
- if (!sharedKey) {
1146
- const q4DecodeKey = findPhaseKernelKey(
1147
- graph,
1148
- graph.decode || [],
1149
- LAYER_PROJECTION_OPS,
1150
- (entry) => entry.kernel === 'fused_matmul_q4.wgsl'
1151
- );
1152
- if (!q4DecodeKey) {
1153
- return null;
1154
- }
1155
- const q4DecodeEntry = result.kernels[q4DecodeKey];
1156
- sharedKey = deriveKernelKey(result.kernels, q4DecodeKey, '_prefill_shared');
1157
- result.kernels[sharedKey] = deriveKernelEntry(
1158
- q4DecodeEntry,
1159
- 'fused_matmul_q4_batched_multicol_shared.wgsl',
1160
- 'main',
1161
- null
1162
- );
1163
- }
1164
-
1165
- let changed = false;
1166
- result.prefill = (result.prefill || []).map((step) => {
1167
- const op = step[0];
1168
- if (!LAYER_PROJECTION_OPS.has(op)) {
1169
- return step;
1170
- }
1171
- const entry = result.kernels[step[1]];
1172
- if (!entry || !DENSE_Q4_PREFILL_FILES.has(entry.kernel)) {
1173
- return step;
1174
- }
1175
-
1176
- const replacementKey = sharedKey;
1177
- if (replacementKey === step[1]) {
1178
- return step;
1179
- }
1180
- changed = true;
1181
- const next = [...step];
1182
- next[1] = replacementKey;
1183
- return next;
1184
- });
1185
-
1186
- return changed ? result : null;
1187
- }
1188
426
 
1189
427
  // =============================================================================
1190
428
  // Transform: remapQ4KPrefillToDense
@@ -1595,74 +833,14 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1595
833
  * numerically aligned with the manifest-owned activation dtype.
1596
834
  *
1597
835
  */
1598
- export function useQwenDecodeF16Matmuls(graph, ctx) {
1599
- const result = cloneGraph(graph);
1600
- let changed = false;
1601
-
1602
- for (const op of ['gate_proj', 'up_proj']) {
1603
- const stepIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === op);
1604
- if (stepIndex === -1) {
1605
- continue;
1606
- }
1607
- const step = result.decode[stepIndex];
1608
- const kernelKey = step[1];
1609
- const kernelEntry = result.kernels[kernelKey];
1610
- if (!kernelEntry) {
1611
- continue;
1612
- }
1613
- const derivedEntry = deriveLinearDecodeF16KernelEntry(kernelEntry);
1614
- if (!derivedEntry) {
1615
- continue;
1616
- }
1617
- const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_decode_f16out');
1618
- result.kernels[derivedKey] = derivedEntry;
1619
- const replacement = [...step];
1620
- replacement[1] = derivedKey;
1621
- result.decode = [
1622
- ...result.decode.slice(0, stepIndex),
1623
- replacement,
1624
- ...result.decode.slice(stepIndex + 1),
1625
- ];
1626
- changed = true;
1627
- }
1628
-
1629
- const postLayerResult = replacePhaseStepKernelKey(
1630
- result.postLayer ?? [],
1631
- 'lm_head',
1632
- (() => {
1633
- const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
1634
- if (!lmHeadStep) {
1635
- return null;
1636
- }
1637
- const lmHeadKernelKey = lmHeadStep[1];
1638
- const lmHeadKernel = result.kernels[lmHeadKernelKey];
1639
- if (!lmHeadKernel) {
1640
- return null;
1641
- }
1642
- const derivedEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
1643
- if (!derivedEntry) {
1644
- return null;
1645
- }
1646
- const derivedKey = deriveKernelKey(result.kernels, lmHeadKernelKey, '_decode_f16out');
1647
- result.kernels[derivedKey] = derivedEntry;
1648
- return derivedKey;
1649
- })()
1650
- );
1651
- if (postLayerResult.changed) {
1652
- result.postLayer = postLayerResult.steps;
1653
- changed = true;
1654
- }
1655
-
1656
- return changed ? result : null;
1657
- }
1658
836
 
1659
837
  // =============================================================================
1660
838
  // Transform: useQwenF16PrimaryMatmuls
1661
839
  // =============================================================================
1662
840
 
1663
841
  /*
1664
- * Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
1665
- * lane when the runtime explicitly requests f16 activations.
842
+ * Select the Qwen 3.5 small-model execution graph's selective f16 probe lane
843
+ * when the runtime explicitly requests f16 activations.
1666
844
  *
1667
845
  * This transform narrows the decode projection and LM-head path while keeping
1668
846
  * `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
@@ -1673,98 +851,6 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
1673
851
  * post-attention RMSNorm).
1674
852
  *
1675
853
  */
1676
- export function useQwenF16PrimaryMatmuls(graph, ctx) {
1677
- const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
1678
- if (!layerTypes || layerTypes.length === 0) {
1679
- return null;
1680
- }
1681
-
1682
- const result = cloneGraph(graph);
1683
- let changed = false;
1684
-
1685
- for (const [phaseName, op] of [['decode', 'attention'], ['prefill', 'attention']]) {
1686
- const phaseSteps = result[phaseName] || [];
1687
- const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
1688
- const kernelKey = step?.[1];
1689
- const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1690
- const derivedEntry = deriveF16AttentionKernelEntry(kernelEntry);
1691
- if (!step || !kernelKey || !derivedEntry) {
1692
- continue;
1693
- }
1694
- const derivedKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f16');
1695
- result.kernels[derivedKey] = derivedEntry;
1696
- const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, derivedKey);
1697
- if (phaseResult.changed) {
1698
- result[phaseName] = phaseResult.steps;
1699
- changed = true;
1700
- }
1701
- }
1702
-
1703
- const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1704
- const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
1705
- const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(decodeProjectionKernel);
1706
- if (decodeProjectionStep && decodeProjectionEntry) {
1707
- const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_primary_f16');
1708
- result.kernels[decodeProjectionKey] = decodeProjectionEntry;
1709
- for (const op of ['q_proj', 'k_proj', 'v_proj', 'gate_proj', 'up_proj']) {
1710
- const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
1711
- if (phaseResult.changed) {
1712
- result.decode = phaseResult.steps;
1713
- changed = true;
1714
- }
1715
- }
1716
- }
1717
-
1718
- const prefillProjectionStep = (result.prefill || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1719
- const prefillProjectionKernel = prefillProjectionStep ? result.kernels[prefillProjectionStep[1]] : null;
1720
- const prefillProjectionEntry = deriveQ4PrefillF16KernelEntry(prefillProjectionKernel);
1721
- if (prefillProjectionStep && prefillProjectionEntry) {
1722
- const prefillProjectionKey = deriveKernelKey(result.kernels, prefillProjectionStep[1], '_primary_f16');
1723
- result.kernels[prefillProjectionKey] = prefillProjectionEntry;
1724
- for (const op of ['q_proj', 'k_proj', 'v_proj']) {
1725
- const phaseResult = replacePhaseStepKernelKey(result.prefill, op, prefillProjectionKey);
1726
- if (phaseResult.changed) {
1727
- result.prefill = phaseResult.steps;
1728
- changed = true;
1729
- }
1730
- }
1731
- }
1732
-
1733
- for (const [phaseName, op] of [['decode', 'o_proj'], ['prefill', 'o_proj']]) {
1734
- const phaseSteps = result[phaseName] || [];
1735
- const step = phaseSteps.find((entry) => Array.isArray(entry) && entry[0] === op);
1736
- const kernelKey = step?.[1];
1737
- const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1738
- if (!step || !kernelKey || !kernelEntry) {
1739
- continue;
1740
- }
1741
- const boundaryKey = deriveKernelKey(result.kernels, kernelKey, '_primary_f32_boundary');
1742
- result.kernels[boundaryKey] = deriveKernelEntryWithPrecision(kernelEntry, {
1743
- inputDtype: 'f32',
1744
- outputDtype: 'f32',
1745
- });
1746
- const phaseResult = replacePhaseStepKernelKey(phaseSteps, op, boundaryKey);
1747
- if (phaseResult.changed) {
1748
- result[phaseName] = phaseResult.steps;
1749
- changed = true;
1750
- }
1751
- }
1752
-
1753
- const lmHeadStep = (result.postLayer || []).find((entry) => Array.isArray(entry) && entry[0] === 'lm_head');
1754
- const lmHeadKernel = lmHeadStep ? result.kernels[lmHeadStep[1]] : null;
1755
- const lmHeadEntry = deriveLmHeadDecodeF16KernelEntry(lmHeadKernel);
1756
- if (lmHeadStep && lmHeadEntry) {
1757
- const lmHeadKey = deriveKernelKey(result.kernels, lmHeadStep[1], '_primary_f16');
1758
- result.kernels[lmHeadKey] = lmHeadEntry;
1759
- const phaseResult = replacePhaseStepKernelKey(result.postLayer, 'lm_head', lmHeadKey);
1760
- if (phaseResult.changed) {
1761
- result.postLayer = phaseResult.steps;
1762
- changed = true;
1763
- }
1764
- }
1765
-
1766
- return changed ? result : null;
1767
- }
1768
854
 
1769
855
  // =============================================================================
1770
856
  // Transform: useQwen36F16Activations
@@ -1774,93 +860,6 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
1774
860
  * Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
1775
861
  *
1776
862
  */
1777
- export function useQwen36F16Activations(graph, ctx) {
1778
- const narrowed = narrowToF16Activations(graph, ctx);
1779
- const result = narrowed ?? cloneGraph(graph);
1780
- let changed = narrowed != null;
1781
-
1782
- const replaceKernelEntry = (key, entry) => {
1783
- if (!key || !entry) {
1784
- return;
1785
- }
1786
- result.kernels[key] = entry;
1787
- changed = true;
1788
- };
1789
- const replaceOps = (phaseName, ops, kernelKey) => {
1790
- for (const op of ops) {
1791
- const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
1792
- if (phaseResult.changed) {
1793
- result[phaseName] = phaseResult.steps;
1794
- changed = true;
1795
- }
1796
- }
1797
- };
1798
-
1799
- const embedStep = findPhaseStep(result.preLayer, 'embed');
1800
- const embedKey = embedStep?.[1] ?? null;
1801
- const embedEntry = embedKey ? result.kernels[embedKey] : null;
1802
- if (embedEntry?.kernel === 'gather_f16.wgsl') {
1803
- replaceKernelEntry(
1804
- embedKey,
1805
- deriveKernelEntryWithPrecision(
1806
- deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
1807
- { inputDtype: 'f16', outputDtype: 'f16' }
1808
- )
1809
- );
1810
- }
1811
-
1812
- const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
1813
- const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
1814
- const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
1815
- if (decodeProjectionEntry) {
1816
- const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
1817
- result.kernels[f16Key] = decodeProjectionEntry;
1818
- replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1819
- }
1820
-
1821
- const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
1822
- const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
1823
- const prefillProjectionEntry =
1824
- deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
1825
- ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
1826
- ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
1827
- if (prefillProjectionEntry) {
1828
- const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
1829
- result.kernels[f16Key] = prefillProjectionEntry;
1830
- replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1831
- }
1832
-
1833
- const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
1834
- const lmHeadKey = lmHeadStep?.[1] ?? null;
1835
- replaceKernelEntry(
1836
- lmHeadKey,
1837
- deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
1838
- );
1839
-
1840
- const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
1841
- const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
1842
- const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
1843
- const lmHeadPrefillF16Entry =
1844
- deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
1845
- ?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
1846
- ?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
1847
- if (lmHeadPrefillF16Entry) {
1848
- replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
1849
- } else if (
1850
- lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
1851
- || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
1852
- ) {
1853
- replaceKernelEntry(
1854
- lmHeadPrefillKey,
1855
- deriveKernelEntryWithPrecision(
1856
- deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
1857
- { inputDtype: 'f16', outputDtype: 'f16' }
1858
- )
1859
- );
1860
- }
1861
-
1862
- return changed ? result : null;
1863
- }
1864
863
 
1865
864
  // =============================================================================
1866
865
  // Transform: useGemma4Int4PleSelectiveF16Decode
@@ -1873,317 +872,11 @@ export function useQwen36F16Activations(graph, ctx) {
1873
872
  * head512 prefill kernels.
1874
873
  *
1875
874
  */
1876
- export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
1877
- const result = cloneGraph(graph);
1878
- let changed = false;
1879
-
1880
- const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1881
- const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
1882
- const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
1883
- if (decodeProjectionStep && decodeProjectionEntry) {
1884
- const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
1885
- result.kernels[decodeProjectionKey] = decodeProjectionEntry;
1886
- for (const op of ['q_proj', 'k_proj', 'v_proj']) {
1887
- const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
1888
- if (phaseResult.changed) {
1889
- result.decode = phaseResult.steps;
1890
- changed = true;
1891
- }
1892
- }
1893
- }
1894
-
1895
- for (const op of ['rope_q', 'rope_k']) {
1896
- const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
1897
- const kernelKey = step?.[1];
1898
- const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1899
- const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
1900
- if (!step || !kernelKey || !replacement) {
1901
- continue;
1902
- }
1903
- const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
1904
- result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
1905
- deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
1906
- { inputDtype: 'f16', outputDtype: 'f16' }
1907
- );
1908
- const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
1909
- if (phaseResult.changed) {
1910
- result.decode = phaseResult.steps;
1911
- changed = true;
1912
- }
1913
- }
1914
-
1915
- const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
1916
- const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
1917
- const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
1918
- if (attentionStep && attentionEntry) {
1919
- const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
1920
- result.kernels[attentionKey] = attentionEntry;
1921
- const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
1922
- if (phaseResult.changed) {
1923
- result.decode = phaseResult.steps;
1924
- changed = true;
1925
- }
1926
- }
1927
-
1928
- const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
1929
- const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
1930
- if (oProjStep && oProjKernel) {
1931
- const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
1932
- result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
1933
- inputDtype: 'f32',
1934
- outputDtype: 'f32',
1935
- });
1936
- const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
1937
- if (phaseResult.changed) {
1938
- result.decode = phaseResult.steps;
1939
- changed = true;
1940
- }
1941
- }
1942
-
1943
- return changed ? result : null;
1944
- }
1945
875
 
1946
876
  // =============================================================================
1947
877
  // Transform: useGemma4TextF16Activations
1948
878
  // =============================================================================
1949
879
 
1950
- const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
1951
- 'q_proj',
1952
- 'k_proj',
1953
- 'v_proj',
1954
- 'gate_proj',
1955
- 'up_proj',
1956
- ]);
1957
-
1958
- function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
1959
- if (!Array.isArray(entries) || !f16Key || !f32Key) {
1960
- return { entries, changed: false };
1961
- }
1962
-
1963
- let changed = false;
1964
- const remapStep = (step) => {
1965
- if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
1966
- return step;
1967
- }
1968
- const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
1969
- ? f32Key
1970
- : f16Key;
1971
- if (step[1] === targetKey) {
1972
- return step;
1973
- }
1974
- const replacement = [...step];
1975
- replacement[1] = targetKey;
1976
- changed = true;
1977
- return replacement;
1978
- };
1979
-
1980
- const nextEntries = entries.map((entry) => {
1981
- if (Array.isArray(entry)) {
1982
- return remapStep(entry);
1983
- }
1984
- if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
1985
- return entry;
1986
- }
1987
- return {
1988
- ...entry,
1989
- steps: entry.steps.map((step) => remapStep(step)),
1990
- };
1991
- });
1992
-
1993
- return { entries: nextEntries, changed };
1994
- }
1995
-
1996
- function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
1997
- let changed = false;
1998
-
1999
- const replaceStepWithSourceEntry = (phaseName, op, precision) => {
2000
- const phase = result[phaseName];
2001
- const sourcePhase = sourceGraph[phaseName];
2002
- const step = findPhaseStep(phase, op);
2003
- const sourceStep = findPhaseStep(sourcePhase, op);
2004
- const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
2005
- const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
2006
- if (!step || !sourceKey || !sourceEntry) {
2007
- return;
2008
- }
2009
- const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
2010
- result.kernels[stableKey] = precision
2011
- ? deriveKernelEntryWithPrecision(sourceEntry, precision)
2012
- : { ...sourceEntry };
2013
- const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
2014
- if (phaseResult.changed) {
2015
- result[phaseName] = phaseResult.steps;
2016
- changed = true;
2017
- }
2018
- };
2019
-
2020
- for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
2021
- replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
2022
- }
2023
- replaceStepWithSourceEntry('decode', 'attention', {
2024
- activationDtype: 'f32',
2025
- kvDtype: 'f16',
2026
- outputDtype: 'f32',
2027
- });
2028
- for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
2029
- replaceStepWithSourceEntry('postLayer', op, null);
2030
- }
2031
-
2032
- return changed;
2033
- }
2034
-
2035
- function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
2036
- const narrowed = narrowToF16Activations(graph, ctx);
2037
- const result = narrowed ?? cloneGraph(graph);
2038
- let changed = narrowed != null;
2039
-
2040
- const replaceKernelEntry = (key, entry) => {
2041
- if (!key || !entry) {
2042
- return;
2043
- }
2044
- result.kernels[key] = entry;
2045
- changed = true;
2046
- };
2047
- const stableTextBoundary = options?.stableTextBoundary === true;
2048
-
2049
- const derivePrefillAttentionEntry = (entry) => {
2050
- if (
2051
- stableTextBoundary
2052
- && typeof entry?.kernel === 'string'
2053
- && entry.kernel === 'attention_small_f16.wgsl'
2054
- ) {
2055
- return deriveKernelEntryWithPrecision(
2056
- deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
2057
- { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2058
- );
2059
- }
2060
- if (
2061
- stableTextBoundary
2062
- && typeof entry?.kernel === 'string'
2063
- && entry.kernel.endsWith('_f16kv.wgsl')
2064
- ) {
2065
- return deriveKernelEntryWithPrecision(
2066
- entry,
2067
- { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2068
- );
2069
- }
2070
- return deriveF16AttentionKernelEntry(entry);
2071
- };
2072
-
2073
- const embedStep = findPhaseStep(result.preLayer, 'embed');
2074
- const embedKey = embedStep?.[1] ?? null;
2075
- const embedEntry = embedKey ? result.kernels[embedKey] : null;
2076
- if (embedEntry?.kernel === 'gather_f16.wgsl') {
2077
- replaceKernelEntry(
2078
- embedKey,
2079
- deriveKernelEntryWithPrecision(
2080
- deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2081
- { inputDtype: 'f16', outputDtype: 'f16' }
2082
- )
2083
- );
2084
- }
2085
-
2086
- const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2087
- const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2088
- replaceKernelEntry(
2089
- decodeProjectionKey,
2090
- deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2091
- );
2092
-
2093
- const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2094
- const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2095
- const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
2096
- const prefillProjectionF16Entry = stableTextBoundary
2097
- ? (
2098
- deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2099
- ?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2100
- ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2101
- )
2102
- : (
2103
- deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2104
- ?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2105
- ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2106
- );
2107
- if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
2108
- const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
2109
- result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
2110
- changed = true;
2111
- const remapped = remapGemma412BPrefillProjectionEntries(
2112
- result.prefill,
2113
- prefillProjectionF16Key,
2114
- prefillProjectionKey
2115
- );
2116
- result.prefill = remapped.entries;
2117
- changed = changed || remapped.changed;
2118
- } else {
2119
- replaceKernelEntry(
2120
- prefillProjectionKey,
2121
- prefillProjectionF16Entry
2122
- );
2123
- }
2124
-
2125
- const replacePrefillAttentionEntries = (entries) => {
2126
- for (const entry of entries || []) {
2127
- if (Array.isArray(entry)) {
2128
- if (entry[0] !== 'attention') {
2129
- continue;
2130
- }
2131
- replaceKernelEntry(
2132
- entry[1],
2133
- derivePrefillAttentionEntry(result.kernels[entry[1]])
2134
- );
2135
- continue;
2136
- }
2137
- if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2138
- replacePrefillAttentionEntries(entry.steps);
2139
- }
2140
- }
2141
- };
2142
- replacePrefillAttentionEntries(result.prefill);
2143
-
2144
- const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2145
- const finalNormKey = finalNormStep?.[1] ?? null;
2146
- const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2147
- if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2148
- replaceKernelEntry(
2149
- finalNormKey,
2150
- deriveKernelEntryWithPrecision(
2151
- deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2152
- { inputDtype: 'f16', outputDtype: 'f16' }
2153
- )
2154
- );
2155
- }
2156
-
2157
- const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2158
- const lmHeadKey = lmHeadStep?.[1] ?? null;
2159
- replaceKernelEntry(
2160
- lmHeadKey,
2161
- deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2162
- );
2163
-
2164
- const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2165
- const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2166
- const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2167
- if (
2168
- lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2169
- || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2170
- ) {
2171
- replaceKernelEntry(
2172
- lmHeadPrefillKey,
2173
- deriveKernelEntryWithPrecision(
2174
- deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2175
- { inputDtype: 'f16', outputDtype: 'f16' }
2176
- )
2177
- );
2178
- }
2179
-
2180
- if (stableTextBoundary) {
2181
- changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
2182
- }
2183
-
2184
- return changed ? result : null;
2185
- }
2186
-
2187
880
  export function useGemma4TextF16Activations(graph, ctx) {
2188
881
  return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
2189
882
  }
@@ -2203,112 +896,13 @@ export function useGemma431BTextF16Activations(graph, ctx) {
2203
896
  /*
2204
897
  * Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
2205
898
  * weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
2206
- * useGemma4TextF16Activations: same Q4 weight pack, kernels narrowed to f16
899
+ * useGemma4TextF16Activations: same Q4 weight capsule, kernels narrowed to f16
2207
900
  * activations, prefill projections promoted from widetile to widetile_f16a,
2208
901
  * decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
2209
902
  * counterparts. Apple Metal stays disabled at the capability layer because the
2210
903
  * fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
2211
904
  *
2212
905
  */
2213
- export function useGemma4Int4PleAf16Activations(graph, ctx) {
2214
- const narrowed = narrowToF16Activations(graph, ctx);
2215
- const result = narrowed ?? cloneGraph(graph);
2216
- let changed = narrowed != null;
2217
-
2218
- const replaceKernelEntry = (key, entry) => {
2219
- if (!key || !entry) {
2220
- return;
2221
- }
2222
- result.kernels[key] = entry;
2223
- changed = true;
2224
- };
2225
-
2226
- const embedStep = findPhaseStep(result.preLayer, 'embed');
2227
- const embedKey = embedStep?.[1] ?? null;
2228
- const embedEntry = embedKey ? result.kernels[embedKey] : null;
2229
- if (embedEntry?.kernel === 'gather_f16.wgsl') {
2230
- replaceKernelEntry(
2231
- embedKey,
2232
- deriveKernelEntryWithPrecision(
2233
- deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2234
- { inputDtype: 'f16', outputDtype: 'f16' }
2235
- )
2236
- );
2237
- }
2238
-
2239
- const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2240
- const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2241
- replaceKernelEntry(
2242
- decodeProjectionKey,
2243
- deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2244
- );
2245
-
2246
- const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2247
- const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2248
- replaceKernelEntry(
2249
- prefillProjectionKey,
2250
- deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2251
- ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
2252
- ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2253
- );
2254
-
2255
- const replacePrefillAttentionEntries = (entries) => {
2256
- for (const entry of entries || []) {
2257
- if (Array.isArray(entry)) {
2258
- if (entry[0] !== 'attention') {
2259
- continue;
2260
- }
2261
- replaceKernelEntry(
2262
- entry[1],
2263
- deriveF16AttentionKernelEntry(result.kernels[entry[1]])
2264
- );
2265
- continue;
2266
- }
2267
- if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2268
- replacePrefillAttentionEntries(entry.steps);
2269
- }
2270
- }
2271
- };
2272
- replacePrefillAttentionEntries(result.prefill);
2273
-
2274
- const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2275
- const finalNormKey = finalNormStep?.[1] ?? null;
2276
- const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2277
- if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2278
- replaceKernelEntry(
2279
- finalNormKey,
2280
- deriveKernelEntryWithPrecision(
2281
- deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2282
- { inputDtype: 'f16', outputDtype: 'f16' }
2283
- )
2284
- );
2285
- }
2286
-
2287
- const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2288
- const lmHeadKey = lmHeadStep?.[1] ?? null;
2289
- replaceKernelEntry(
2290
- lmHeadKey,
2291
- deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2292
- );
2293
-
2294
- const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2295
- const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2296
- const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2297
- if (
2298
- lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2299
- || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2300
- ) {
2301
- replaceKernelEntry(
2302
- lmHeadPrefillKey,
2303
- deriveKernelEntryWithPrecision(
2304
- deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2305
- { inputDtype: 'f16', outputDtype: 'f16' }
2306
- )
2307
- );
2308
- }
2309
-
2310
- return changed ? result : null;
2311
- }
2312
906
 
2313
907
  // =============================================================================
2314
908
  // Composition