doppler-gpu 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1435) hide show
  1. package/CHANGELOG.md +100 -38
  2. package/README.md +47 -195
  3. package/models/catalog.json +108 -51
  4. package/package.json +92 -122
  5. package/src/capsule-runtime.d.ts +15 -8
  6. package/src/capsule-runtime.js +7 -3
  7. package/src/cli/cli-output.js +85 -9
  8. package/src/cli/doppler-cli.js +23 -21
  9. package/src/client/capsule-host.browser.js +6 -0
  10. package/src/client/capsule-host.d.ts +11 -0
  11. package/src/client/capsule-host.js +6 -0
  12. package/src/client/doppler-api.browser.js +1 -1
  13. package/src/client/doppler-api.d.ts +1 -0
  14. package/src/client/doppler-api.js +1 -1
  15. package/src/client/doppler-registry.js +6 -0
  16. package/src/client/inspection.d.ts +10 -1
  17. package/src/client/inspection.js +24 -12
  18. package/src/client/model-host/index.d.ts +3 -0
  19. package/src/client/model-host/index.js +73 -18
  20. package/src/client/model-host/model-session.d.ts +41 -17
  21. package/src/client/model-host/model-session.js +283 -154
  22. package/src/client/model-host/source-loading.js +910 -0
  23. package/src/client/provider.js +6 -6
  24. package/src/client/resident-partitions.d.ts +30 -0
  25. package/src/client/resident-partitions.js +114 -0
  26. package/src/client/runtime/capsule-acquisition.d.ts +17 -2
  27. package/src/client/runtime/capsule-artifact-source.js +23 -1
  28. package/src/client/runtime/capsule-operation-adapters.d.ts +3 -2
  29. package/src/client/runtime/capsule-operation-adapters.js +44 -8
  30. package/src/client/runtime/capsule-operation-deltas.js +44 -0
  31. package/src/client/runtime/capsule-operation-executor.d.ts +2 -2
  32. package/src/client/runtime/capsule-operation-executor.js +11 -5
  33. package/src/client/runtime/capsule-operation-stream.d.ts +10 -0
  34. package/src/client/runtime/capsule-operation-stream.js +81 -0
  35. package/src/client/runtime/capsule-piece-reader.js +88 -0
  36. package/src/client/runtime/capsule-program-adapter.js +51 -21
  37. package/src/client/runtime/command-executor.d.ts +13 -0
  38. package/src/client/runtime/command-executor.js +54 -11
  39. package/src/client/runtime/composition-root.d.ts +35 -12
  40. package/src/client/runtime/composition-root.js +186 -57
  41. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +3 -2
  42. package/src/client/runtime/fetch-capsule-artifact-store.js +56 -1
  43. package/src/client/runtime/lora.d.ts +22 -0
  44. package/src/client/runtime/lora.js +86 -26
  45. package/src/client/runtime/model-manager.js +10 -75
  46. package/src/client/runtime/model-source.d.ts +1 -0
  47. package/src/client/runtime/model-source.js +1 -1
  48. package/src/client/runtime/node-quickstart-cache.js +7 -2
  49. package/src/client/runtime/resource-binder.d.ts +1 -0
  50. package/src/client/runtime/resource-binder.js +20 -14
  51. package/src/client/runtime/scoped-session.d.ts +10 -0
  52. package/src/client/runtime/scoped-session.js +41 -3
  53. package/src/client/runtime/session-controller.d.ts +15 -1
  54. package/src/client/runtime/session-controller.js +102 -30
  55. package/src/client/runtime/verified-capsule-artifact-store.d.ts +27 -0
  56. package/src/client/runtime/verified-capsule-artifact-store.js +255 -58
  57. package/src/client/wrap-pipeline-handle.js +23 -17
  58. package/src/config/canonical-contract-codec.d.ts +10 -0
  59. package/src/config/canonical-contract-codec.js +144 -0
  60. package/src/config/canonical-generation-policy.d.ts +7 -0
  61. package/src/config/canonical-generation-policy.js +66 -0
  62. package/src/config/capsule-loading.d.ts +5 -1
  63. package/src/config/capsule-loading.js +1 -1
  64. package/src/config/capsule-loading.schema.json +63 -4
  65. package/src/config/capsule-operation.d.ts +10 -3
  66. package/src/config/capsule-operation.js +6 -1
  67. package/src/config/capsule-operations.json +26 -1
  68. package/src/config/capsule-release-contract.js +2 -29
  69. package/src/config/capsule-token-selection.d.ts +15 -0
  70. package/src/config/capsule-token-selection.js +39 -0
  71. package/src/config/capsule-token-selection.json +11 -0
  72. package/src/config/capsule-v2.d.ts +1 -0
  73. package/src/config/capsule-v2.js +14 -21
  74. package/src/config/choice-scoring-reference.js +66 -0
  75. package/src/config/choice-scoring.d.ts +21 -0
  76. package/src/config/choice-scoring.js +66 -0
  77. package/src/config/choice-scoring.json +8 -0
  78. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +4 -4
  79. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +7 -7
  80. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  81. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +12 -8
  82. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +260 -55
  83. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -10
  84. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +9 -8
  85. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +6 -6
  86. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +9 -9
  87. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +8 -7
  88. package/src/config/conversion/gemma3/gemma-3-270m-it-w8a8-ehf16-af32.json +491 -0
  89. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +5 -5
  90. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +9 -8
  91. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +24 -8
  92. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +24 -8
  93. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +25 -9
  94. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +21 -5
  95. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +26 -9
  96. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +26 -9
  97. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +26 -9
  98. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +25 -8
  99. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +25 -8
  100. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +7 -6
  101. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +7 -7
  102. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +7 -6
  103. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +7 -6
  104. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +20 -12
  105. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +7 -9
  106. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +7 -7
  107. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +7 -7
  108. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +252 -53
  109. package/src/config/conversion/qwen3/qwen-3-4b-instruct-2507-q4k-ehf16-af32.json +501 -0
  110. package/src/config/conversion/qwen3/qwen-3-5-0-8b-canonical-f32.json +461 -0
  111. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -8
  112. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -8
  113. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +7 -7
  114. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +8 -8
  115. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +8 -8
  116. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -8
  117. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-q4_0-passthrough.json +1218 -0
  118. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-text-q4k-ehf16-af32.json +581 -0
  119. package/src/config/conversion/qwen3/qwen-3-8-flash-next-text-q4k-ehf16-af32.json +1063 -0
  120. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +11 -11
  121. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +6 -6
  122. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +18 -10
  123. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +10 -10
  124. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +18 -18
  125. package/src/config/conversion/qwen3/qwen3.8-27b-q4_0-passthrough.json +823 -0
  126. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +11 -11
  127. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +11 -11
  128. package/src/config/embedding-contract.d.ts +17 -0
  129. package/src/config/embedding-contract.js +12 -0
  130. package/src/config/embedding-reference.js +32 -0
  131. package/src/config/execution-registry-contract.d.ts +41 -0
  132. package/src/config/execution-registry-contract.js +171 -0
  133. package/src/config/generation-contract.d.ts +28 -1
  134. package/src/config/generation-contract.json +119 -0
  135. package/src/config/initial-execution-identity.d.ts +1 -1
  136. package/src/config/initial-execution-identity.js +9 -9
  137. package/src/config/inspection/activity-stages.json +14 -0
  138. package/src/config/inspection/observation-policies.json +123 -0
  139. package/src/config/kernel-path-loader.d.ts +30 -0
  140. package/src/config/kernel-path-loader.js +69 -13
  141. package/src/config/kernel-registry-contract.d.ts +78 -13
  142. package/src/config/kernel-registry-contract.js +110 -38
  143. package/src/config/kernels/backward-registry.json +1 -1
  144. package/src/config/kernels/registry.js +3 -47
  145. package/src/config/kernels/registry.json +31747 -7849
  146. package/src/config/layer-partition-plan.d.ts +93 -0
  147. package/src/config/layer-partition-plan.js +305 -0
  148. package/src/config/manifest-inference-validation.js +10 -0
  149. package/src/config/merge/execution-patches.js +14 -0
  150. package/src/config/merge.d.ts +4 -0
  151. package/src/config/merge.js +50 -0
  152. package/src/config/model-ir-v2.d.ts +1 -0
  153. package/src/config/model-ir-v2.js +5 -16
  154. package/src/config/model-ir.js +9 -14
  155. package/src/config/ordered-resident-qualification.d.ts +16 -0
  156. package/src/config/ordered-resident-qualification.js +62 -0
  157. package/src/config/param-categories.js +4 -0
  158. package/src/config/param-validator.js +64 -29
  159. package/src/config/production-release-evidence.js +2 -27
  160. package/src/config/production-release.js +2 -29
  161. package/src/config/quickstart-registry.json +57 -8
  162. package/src/config/reasoning-output.d.ts +6 -0
  163. package/src/config/reasoning-output.js +26 -0
  164. package/src/config/revocation-policy.js +1 -4
  165. package/src/config/rope-frequencies.js +9 -0
  166. package/src/config/runtime/profiles/demo-f16-comparison.json +20 -0
  167. package/src/config/runtime/profiles/flash-next-node-host-cache-qualification.json +21 -0
  168. package/src/config/runtime/profiles/flash-next-paged-qualification.json +23 -0
  169. package/src/config/runtime/profiles/flash-next-routed-f32kv-qualification.json +72 -0
  170. package/src/config/runtime/profiles/flash-next-routed-qualification.json +27 -0
  171. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +51 -2
  172. package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-parity.json +88 -0
  173. package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8.json +173 -0
  174. package/src/config/runtime/profiles/qwen-3-5-0-8b-vulkan-throughput.json +33 -0
  175. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +51 -2
  176. package/src/config/runtime/profiles/qwen-3-5-2b-vulkan-parity.json +15 -0
  177. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-candidate.json +4491 -0
  178. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +4407 -0
  179. package/src/config/runtime/profiles/qwen3-6-35b-a3b-route-f32-candidate.json +17 -0
  180. package/src/config/runtime/profiles/qwen3-6-35b-a3b-route-f32-workspace-candidate.json +13 -0
  181. package/src/config/runtime.d.ts +6 -0
  182. package/src/config/runtime.js +51 -21
  183. package/src/config/schema/benchmark.schema.d.ts +1 -0
  184. package/src/config/schema/benchmark.schema.js +2 -0
  185. package/src/config/schema/conversion.schema.d.ts +5 -0
  186. package/src/config/schema/converter.schema.d.ts +10 -0
  187. package/src/config/schema/converter.schema.js +10 -0
  188. package/src/config/schema/debug.schema.d.ts +1 -0
  189. package/src/config/schema/doppler.schema.js +16 -9
  190. package/src/config/schema/emulation.schema.js +0 -77
  191. package/src/config/schema/execution-v1.schema.d.ts +32 -2
  192. package/src/config/schema/execution-v1.schema.js +8 -0
  193. package/src/config/schema/fp8.schema.d.ts +17 -0
  194. package/src/config/schema/fp8.schema.js +27 -0
  195. package/src/config/schema/hyper-connection-decoder.schema.d.ts +98 -0
  196. package/src/config/schema/hyper-connection-decoder.schema.js +213 -0
  197. package/src/config/schema/index.d.ts +2 -2
  198. package/src/config/schema/index.js +3 -0
  199. package/src/config/schema/inference.schema.d.ts +8 -2
  200. package/src/config/schema/kernel-registry.schema.d.ts +22 -7
  201. package/src/config/schema/kernel-registry.schema.js +13 -2
  202. package/src/config/schema/kernel-thresholds.schema.d.ts +3 -0
  203. package/src/config/schema/kernel-thresholds.schema.js +14 -3
  204. package/src/config/schema/kernel-warmup.schema.d.ts +15 -0
  205. package/src/config/schema/kernel-warmup.schema.js +3 -0
  206. package/src/config/schema/loading.schema.d.ts +41 -0
  207. package/src/config/schema/loading.schema.js +67 -0
  208. package/src/config/schema/manifest-rope.schema.d.ts +55 -0
  209. package/src/config/schema/manifest.schema.d.ts +47 -57
  210. package/src/config/schema/manifest.schema.js +41 -2
  211. package/src/config/schema/ngram-embedding.schema.d.ts +116 -0
  212. package/src/config/schema/ngram-embedding.schema.js +312 -0
  213. package/src/config/schema/program-bundle.schema.d.ts +1 -0
  214. package/src/config/schema/program-bundle.schema.js +8 -3
  215. package/src/config/schema/quantization-defaults.schema.d.ts +1 -1
  216. package/src/config/schema/quantization.schema.d.ts +2 -0
  217. package/src/config/schema/quantization.schema.js +2 -0
  218. package/src/config/schema/speculation-self.schema.d.ts +6 -1
  219. package/src/config/schema/speculation-self.schema.js +29 -2
  220. package/src/config/schema/w8a8.schema.d.ts +24 -0
  221. package/src/config/schema/w8a8.schema.js +38 -0
  222. package/src/config/source-packages/litert/gemma-4-e2b-it.json +6 -6
  223. package/src/config/supported-operations.json +105 -0
  224. package/src/config/target-plan.d.ts +16 -2
  225. package/src/config/target-plan.js +63 -16
  226. package/src/config/wgsl-language-contract.js +47 -0
  227. package/src/converter/artifact-identity.js +27 -47
  228. package/src/converter/capability-transform-resolver.js +80 -0
  229. package/src/converter/conversion-plan.js +14 -14
  230. package/src/converter/core.d.ts +2 -0
  231. package/src/converter/core.js +15 -16
  232. package/src/converter/execution-graph/fusion.js +343 -0
  233. package/src/converter/execution-graph/kv-storage.js +14 -0
  234. package/src/converter/execution-graph/validation.js +431 -0
  235. package/src/converter/execution-graph-transforms.js +927 -0
  236. package/src/converter/forge-candidate-evaluation.js +1 -1
  237. package/src/converter/forge-qualification.js +101 -30
  238. package/src/converter/forge-stages.js +107 -102
  239. package/src/converter/fp8-source.js +99 -0
  240. package/src/converter/linear-norm-gate.js +115 -0
  241. package/src/converter/manifest-builder.js +20 -6
  242. package/src/converter/parsers/gguf.js +21 -2
  243. package/src/converter/quantization-info.js +56 -5
  244. package/src/converter/quantizer.js +4 -159
  245. package/src/converter/qwen4-exp-source-defaults.json +35 -0
  246. package/src/converter/qwen4-exp-source.js +293 -0
  247. package/src/converter/resident-qualification.js +58 -0
  248. package/src/converter/retained-source-shards.js +97 -0
  249. package/src/converter/rope-config.js +1 -0
  250. package/src/converter/source-rotary-frequencies.d.ts +12 -0
  251. package/src/converter/source-rotary-frequencies.js +34 -0
  252. package/src/converter/source-tensor-rules.d.ts +25 -1
  253. package/src/converter/source-tensor-rules.js +31 -1
  254. package/src/converter/tensor-transform.js +60 -7
  255. package/src/converter/tokenizer-contract.js +1 -3
  256. package/src/converter/w8a8-quantizer.js +56 -0
  257. package/src/debug/activation-trace.d.ts +34 -0
  258. package/src/debug/activation-trace.js +89 -0
  259. package/src/debug/activity-trace.d.ts +52 -0
  260. package/src/debug/activity-trace.js +127 -0
  261. package/src/debug/capture-policy.d.ts +9 -1
  262. package/src/debug/capture-policy.js +52 -21
  263. package/src/debug/index.d.ts +3 -0
  264. package/src/debug/index.js +12 -6
  265. package/src/debug/log.d.ts +6 -0
  266. package/src/debug/log.js +26 -0
  267. package/src/debug/perf-guards.js +71 -0
  268. package/src/debug/pipeline-config.js +141 -0
  269. package/src/debug/stats.d.ts +1 -1
  270. package/src/debug/stats.js +7 -2
  271. package/src/debug/tensor.d.ts +15 -3
  272. package/src/debug/tensor.js +43 -27
  273. package/src/debug/trace.js +12 -0
  274. package/src/errors/doppler-error.js +6 -2
  275. package/src/experimental/audio/gemma4.js +19 -10
  276. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +1 -1
  277. package/src/experimental/diffusion/pipeline.d.ts +1 -0
  278. package/src/experimental/distribution/shard-delivery/receipt.js +3 -73
  279. package/src/experimental/distribution/shard-delivery/retry.js +6 -112
  280. package/src/experimental/distribution/shard-delivery.js +19 -632
  281. package/src/experimental/energy/pipeline.d.ts +1 -0
  282. package/src/experimental/logits/cpu.js +2 -7
  283. package/src/experimental/training/artifacts.js +1 -4
  284. package/src/experimental/training/attention-forward.js +1 -1
  285. package/src/experimental/training/autograd.d.ts +21 -0
  286. package/src/experimental/training/autograd.js +44 -20
  287. package/src/experimental/training/checkpoint.js +1 -4
  288. package/src/experimental/training/clip.js +1 -1
  289. package/src/experimental/training/distillation/student-fixture/execution.js +9 -12
  290. package/src/experimental/training/distillation/student-fixture/model.js +9 -12
  291. package/src/experimental/training/distillation/student-fixture.js +9 -12
  292. package/src/experimental/training/lora/checkpoint.js +1 -1
  293. package/src/experimental/training/lora/execution.js +1 -1
  294. package/src/experimental/training/lora/plan.js +1 -1
  295. package/src/experimental/training/lora/recovery.js +1 -1
  296. package/src/experimental/training/lora-pipeline.js +9 -83
  297. package/src/experimental/training/lora-training-round.d.ts +39 -0
  298. package/src/experimental/training/lora-training-round.js +90 -0
  299. package/src/experimental/training/lora.js +2 -1
  300. package/src/experimental/training/loss-scaling.js +1 -1
  301. package/src/experimental/training/loss.js +3 -1
  302. package/src/experimental/training/native-qwen-lora.d.ts +25 -6
  303. package/src/experimental/training/native-qwen-lora.js +176 -21
  304. package/src/experimental/training/operator-command.js +1 -49
  305. package/src/experimental/training/parameter-groups.js +39 -0
  306. package/src/experimental/training/report-ranking.js +52 -0
  307. package/src/experimental/training/runner/lifecycle.js +4 -61
  308. package/src/experimental/training/runner.js +2 -5
  309. package/src/experimental/training/suite/execution.js +3 -5
  310. package/src/experimental/training/suite/plan.js +6 -21
  311. package/src/experimental/training/suite.js +5 -34
  312. package/src/experimental/training/trainer.js +2 -36
  313. package/src/experimental/training/workload-capsules/registry.json +473 -0
  314. package/src/experimental/turboquant/codebook.js +0 -29
  315. package/src/formats/contract-validation.js +30 -0
  316. package/src/formats/flatbuffer-tables.js +55 -0
  317. package/src/formats/float16.js +81 -0
  318. package/src/formats/float8.js +19 -0
  319. package/src/formats/gguf/block-storage.js +51 -0
  320. package/src/formats/gguf/quantized-formats.json +10 -0
  321. package/src/formats/gguf/qwen-source.js +88 -0
  322. package/src/formats/gguf/qwen-source.json +8 -0
  323. package/src/formats/gguf/tokenizer-pre.json +4 -0
  324. package/src/formats/gguf/tokenizer.js +50 -0
  325. package/src/formats/litert/types.js +8 -180
  326. package/src/formats/paged-source-reader.js +158 -0
  327. package/src/formats/q4k-dequant.js +95 -0
  328. package/src/formats/rdrr/classification.js +4 -0
  329. package/src/formats/rdrr/distributed/validation.js +1 -4
  330. package/src/formats/rdrr/fp8-contract.js +38 -0
  331. package/src/formats/rdrr/fp8-storage.js +83 -0
  332. package/src/formats/rdrr/functional-descriptor.js +1 -4
  333. package/src/formats/rdrr/parsing.js +1 -1
  334. package/src/formats/rdrr/q4_0-storage.js +24 -0
  335. package/src/formats/rdrr/storage-descriptor.js +7 -1
  336. package/src/formats/rdrr/types.d.ts +7 -1
  337. package/src/formats/rdrr/validation.js +37 -6
  338. package/src/formats/rdrr/w8a8-storage.js +64 -0
  339. package/src/formats/safetensors/types.js +4 -0
  340. package/src/formats/sha256.js +104 -103
  341. package/src/formats/sorted-json.js +9 -0
  342. package/src/formats/source-runtime.d.ts +52 -0
  343. package/src/formats/tflite/types.js +7 -161
  344. package/src/generation/index.d.ts +7 -1
  345. package/src/generation/index.js +7 -1
  346. package/src/gpu/buffer-identity.js +4 -0
  347. package/src/gpu/command-recorder.d.ts +66 -29
  348. package/src/gpu/command-recorder.js +157 -161
  349. package/src/gpu/device-bindings.js +98 -0
  350. package/src/gpu/device-state.js +0 -12
  351. package/src/gpu/device.d.ts +9 -0
  352. package/src/gpu/device.js +104 -141
  353. package/src/gpu/kernel-selector.js +73 -12
  354. package/src/gpu/kernels/attention/executors-components.js +235 -0
  355. package/src/gpu/kernels/attention/executors.js +296 -510
  356. package/src/gpu/kernels/attention/plan.js +65 -18
  357. package/src/gpu/kernels/attention/query-splitk.js +80 -0
  358. package/src/gpu/kernels/attention_decode_gqa_splitk.wgsl +197 -0
  359. package/src/gpu/kernels/attention_decode_online.wgsl +348 -0
  360. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +126 -22
  361. package/src/gpu/kernels/attention_decode_subgroup.wgsl +5 -6
  362. package/src/gpu/kernels/attention_head256_f16kv.wgsl +56 -9
  363. package/src/gpu/kernels/attention_indexed.wgsl +179 -0
  364. package/src/gpu/kernels/attention_indexed_f16kv.wgsl +180 -0
  365. package/src/gpu/kernels/attention_paired.wgsl +361 -0
  366. package/src/gpu/kernels/attention_paired_tiled.wgsl +383 -0
  367. package/src/gpu/kernels/attention_prefill_partitioned_head256_f16kv.wgsl +247 -0
  368. package/src/gpu/kernels/attention_prefill_query_splitk_f32.wgsl +234 -0
  369. package/src/gpu/kernels/attention_prefill_query_splitk_reduce_f32.wgsl +103 -0
  370. package/src/gpu/kernels/attention_prefill_query_tile_f16kv.wgsl +225 -0
  371. package/src/gpu/kernels/attention_prefill_query_tile_f32.wgsl +222 -0
  372. package/src/gpu/kernels/backward/adam.js +11 -3
  373. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +51 -15
  374. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +9 -8
  375. package/src/gpu/kernels/backward/gated_delta_recurrent_checkpoint_forward.js +149 -0
  376. package/src/gpu/kernels/backward/gated_delta_recurrent_checkpoint_forward.wgsl +109 -0
  377. package/src/gpu/kernels/backward/gated_delta_recurrent_checkpointed_backward.js +120 -0
  378. package/src/gpu/kernels/backward/gelu_backward.js +8 -12
  379. package/src/gpu/kernels/backward/index.js +6 -0
  380. package/src/gpu/kernels/backward/qwen_attention_split_q_gate_backward.js +80 -0
  381. package/src/gpu/kernels/backward/qwen_attention_split_q_gate_backward.wgsl +26 -0
  382. package/src/gpu/kernels/backward/qwen_gqa_attention_backward.js +157 -0
  383. package/src/gpu/kernels/backward/qwen_gqa_attention_backward.wgsl +82 -0
  384. package/src/gpu/kernels/backward/qwen_gqa_softmax_backward_scores.wgsl +73 -0
  385. package/src/gpu/kernels/backward/qwen_gqa_softmax_recompute.wgsl +97 -0
  386. package/src/gpu/kernels/backward/qwen_linear_attention_prepare_backward.js +221 -0
  387. package/src/gpu/kernels/backward/qwen_linear_attention_prepare_backward.wgsl +136 -0
  388. package/src/gpu/kernels/backward/rmsnorm_backward.js +98 -80
  389. package/src/gpu/kernels/backward/rope_backward.js +47 -10
  390. package/src/gpu/kernels/backward/rope_backward.wgsl +46 -22
  391. package/src/gpu/kernels/backward/sigmoid_gated_backward.js +83 -0
  392. package/src/gpu/kernels/backward/sigmoid_gated_backward.wgsl +49 -0
  393. package/src/gpu/kernels/backward/silu_backward.js +8 -12
  394. package/src/gpu/kernels/backward/softmax_backward.js +8 -12
  395. package/src/gpu/kernels/cast_f32_to_two_f32.wgsl +12 -0
  396. package/src/gpu/kernels/cast_two_f32_to_f16.wgsl +49 -0
  397. package/src/gpu/kernels/causal_conv1d_silu.js +80 -0
  398. package/src/gpu/kernels/causal_conv1d_silu.wgsl +44 -0
  399. package/src/gpu/kernels/check-stop.js +1 -1
  400. package/src/gpu/kernels/check_finiteness_f32.wgsl +33 -0
  401. package/src/gpu/kernels/conv2d.js +30 -42
  402. package/src/gpu/kernels/dequant.js +1 -1
  403. package/src/gpu/kernels/dequant_mxfp4.wgsl +6 -4
  404. package/src/gpu/kernels/diffusion-gemma-sampling.js +26 -7
  405. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +66 -2
  406. package/src/gpu/kernels/dispatch.js +36 -1
  407. package/src/gpu/kernels/embedding-pool.js +15 -13
  408. package/src/gpu/kernels/feature-check.js +7 -2
  409. package/src/gpu/kernels/fp8.js +70 -0
  410. package/src/gpu/kernels/fused_ffn.js +83 -154
  411. package/src/gpu/kernels/fused_ffn_q4k.wgsl +43 -1
  412. package/src/gpu/kernels/fused_ffn_q4k_subgroup8.wgsl +228 -0
  413. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +2 -2
  414. package/src/gpu/kernels/fused_matmul_q4_subgroup.wgsl +146 -0
  415. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +14 -4
  416. package/src/gpu/kernels/fused_matmul_q4_widetile_direct.wgsl +204 -0
  417. package/src/gpu/kernels/fused_matmul_residual.js +19 -72
  418. package/src/gpu/kernels/fused_matmul_rmsnorm.js +34 -127
  419. package/src/gpu/kernels/gated-delta-norm-gate.js +45 -0
  420. package/src/gpu/kernels/gated_delta_conv.wgsl +57 -6
  421. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +57 -6
  422. package/src/gpu/kernels/gated_delta_conv_paired.wgsl +282 -0
  423. package/src/gpu/kernels/gated_delta_conv_prefill.wgsl +144 -0
  424. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +375 -40
  425. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +375 -40
  426. package/src/gpu/kernels/gated_delta_fused_decode_paired.wgsl +511 -0
  427. package/src/gpu/kernels/gated_delta_fused_decode_paired_f16.wgsl +513 -0
  428. package/src/gpu/kernels/gated_delta_fused_decode_paired_sigmoid_gate.wgsl +514 -0
  429. package/src/gpu/kernels/gated_delta_fused_decode_paired_sigmoid_gate_f16.wgsl +516 -0
  430. package/src/gpu/kernels/gated_delta_fused_decode_register.wgsl +750 -0
  431. package/src/gpu/kernels/gated_delta_fused_decode_sigmoid_gate.wgsl +511 -0
  432. package/src/gpu/kernels/gated_delta_fused_decode_sigmoid_gate_f16.wgsl +513 -0
  433. package/src/gpu/kernels/gated_delta_gate_paired.wgsl +271 -0
  434. package/src/gpu/kernels/gated_delta_gate_sigmoid_paired.wgsl +273 -0
  435. package/src/gpu/kernels/gated_delta_norm_paired.wgsl +288 -0
  436. package/src/gpu/kernels/gated_delta_prefill_norm_gate.wgsl +227 -0
  437. package/src/gpu/kernels/gated_delta_qk_paired.wgsl +297 -0
  438. package/src/gpu/kernels/gated_delta_qk_paired_parallel.wgsl +299 -0
  439. package/src/gpu/kernels/gated_delta_recurrent.wgsl +362 -32
  440. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +362 -32
  441. package/src/gpu/kernels/gated_delta_recurrent_gguf.wgsl +496 -0
  442. package/src/gpu/kernels/gated_delta_recurrent_gguf_register_prefill.wgsl +727 -0
  443. package/src/gpu/kernels/gated_delta_recurrent_paired.wgsl +337 -0
  444. package/src/gpu/kernels/gated_delta_recurrent_register_prefill.wgsl +546 -0
  445. package/src/gpu/kernels/gated_delta_recurrent_sigmoid_gate.wgsl +498 -0
  446. package/src/gpu/kernels/gated_delta_recurrent_sigmoid_gate_f16.wgsl +500 -0
  447. package/src/gpu/kernels/gated_delta_state_paired.wgsl +310 -0
  448. package/src/gpu/kernels/gated_delta_state_paired_sliced.wgsl +348 -0
  449. package/src/gpu/kernels/gated_delta_state_paired_sliced_rolled.wgsl +353 -0
  450. package/src/gpu/kernels/gated_rmsnorm.js +74 -0
  451. package/src/gpu/kernels/gated_rmsnorm.wgsl +62 -0
  452. package/src/gpu/kernels/gather.js +81 -46
  453. package/src/gpu/kernels/gather.wgsl +3 -2
  454. package/src/gpu/kernels/gather_bf16.wgsl +3 -2
  455. package/src/gpu/kernels/gather_f16.wgsl +3 -2
  456. package/src/gpu/kernels/gather_f16_f16_out.wgsl +3 -2
  457. package/src/gpu/kernels/gather_f16_out.wgsl +3 -2
  458. package/src/gpu/kernels/gather_f16_vec4.wgsl +3 -2
  459. package/src/gpu/kernels/gather_f16_vec4_f16_out.wgsl +3 -2
  460. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +3 -2
  461. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +3 -2
  462. package/src/gpu/kernels/gather_q4_0.wgsl +57 -0
  463. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +3 -2
  464. package/src/gpu/kernels/gather_split4_f16_vec4_f32_out.wgsl +104 -0
  465. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +3 -2
  466. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +3 -2
  467. package/src/gpu/kernels/gather_vec4.wgsl +3 -2
  468. package/src/gpu/kernels/gather_vec4_f16_out.wgsl +3 -2
  469. package/src/gpu/kernels/gelu.js +5 -12
  470. package/src/gpu/kernels/gemma4-route-expert.js +2 -5
  471. package/src/gpu/kernels/generated/uniform-writers-0.js +600 -0
  472. package/src/gpu/kernels/generated/uniform-writers-1.js +602 -0
  473. package/src/gpu/kernels/generated/uniform-writers-2.js +516 -0
  474. package/src/gpu/kernels/generated/uniform-writers.js +557 -0
  475. package/src/gpu/kernels/hyper-connection.js +240 -0
  476. package/src/gpu/kernels/hyper_connection_activation.wgsl +110 -0
  477. package/src/gpu/kernels/hyper_connection_branch.wgsl +60 -0
  478. package/src/gpu/kernels/hyper_connection_inject.wgsl +49 -0
  479. package/src/gpu/kernels/hyper_connection_norm.wgsl +72 -0
  480. package/src/gpu/kernels/kernel-bindings.js +37 -0
  481. package/src/gpu/kernels/kernel-configs.d.ts +12 -2
  482. package/src/gpu/kernels/kernel-configs.js +40 -23
  483. package/src/gpu/kernels/kernel-execution.js +87 -26
  484. package/src/gpu/kernels/kernel-geometry.js +43 -0
  485. package/src/gpu/kernels/kernel-prewarm.js +18 -15
  486. package/src/gpu/kernels/kv-cache-write.js +4 -6
  487. package/src/gpu/kernels/layernorm.js +7 -45
  488. package/src/gpu/kernels/linear-attention-core.js +247 -119
  489. package/src/gpu/kernels/linear-attention-recovery.js +90 -0
  490. package/src/gpu/kernels/linear-norm-ab.js +80 -0
  491. package/src/gpu/kernels/linear_norm_ab_f32.wgsl +243 -0
  492. package/src/gpu/kernels/logit-finalize.js +9 -3
  493. package/src/gpu/kernels/logit-scatter.js +9 -3
  494. package/src/gpu/kernels/logit-suppress.js +24 -0
  495. package/src/gpu/kernels/logit_suppress.wgsl +19 -0
  496. package/src/gpu/kernels/matmul-dispatch.js +70 -146
  497. package/src/gpu/kernels/matmul-selection.js +75 -18
  498. package/src/gpu/kernels/matmul-workgroup-capacity.js +37 -0
  499. package/src/gpu/kernels/matmul.js +81 -108
  500. package/src/gpu/kernels/matmul_compensated_f32.wgsl +215 -0
  501. package/src/gpu/kernels/matmul_f16w_f32a_tiled.wgsl +59 -46
  502. package/src/gpu/kernels/matmul_f32.wgsl +23 -17
  503. package/src/gpu/kernels/matmul_f32_row_subgroup.wgsl +88 -0
  504. package/src/gpu/kernels/matmul_fp8.wgsl +60 -0
  505. package/src/gpu/kernels/matmul_gemv_compensated_f32.wgsl +211 -0
  506. package/src/gpu/kernels/matmul_gemv_f32.wgsl +63 -0
  507. package/src/gpu/kernels/matmul_gemv_f32_subgroup.wgsl +84 -0
  508. package/src/gpu/kernels/matmul_gguf_columns_hoisted_f32.wgsl +342 -0
  509. package/src/gpu/kernels/matmul_gguf_pair_columns_f32.wgsl +279 -0
  510. package/src/gpu/kernels/matmul_gguf_pair_f32.wgsl +197 -0
  511. package/src/gpu/kernels/matmul_paired.wgsl +315 -0
  512. package/src/gpu/kernels/matmul_paired_compensated.wgsl +334 -0
  513. package/src/gpu/kernels/matmul_q4_0.wgsl +234 -0
  514. package/src/gpu/kernels/matmul_q4_0_tile128.wgsl +477 -0
  515. package/src/gpu/kernels/matmul_q4_0_tile128_bounded.wgsl +483 -0
  516. package/src/gpu/kernels/matmul_q4_0_tile64.wgsl +282 -0
  517. package/src/gpu/kernels/matmul_q4_0_tile64_bounded.wgsl +288 -0
  518. package/src/gpu/kernels/matmul_q4_0_tile64x128.wgsl +364 -0
  519. package/src/gpu/kernels/matmul_q4_0_tile64x128_bounded.wgsl +370 -0
  520. package/src/gpu/kernels/matmul_q4_1_columns_f32.wgsl +132 -0
  521. package/src/gpu/kernels/matmul_q5k.wgsl +181 -0
  522. package/src/gpu/kernels/matmul_q5k_columns.wgsl +194 -0
  523. package/src/gpu/kernels/matmul_q5k_cooperative.wgsl +129 -0
  524. package/src/gpu/kernels/matmul_q5k_tile64.wgsl +309 -0
  525. package/src/gpu/kernels/matmul_w8a8.wgsl +55 -0
  526. package/src/gpu/kernels/matmul_w8a8_packed.wgsl +54 -0
  527. package/src/gpu/kernels/matmul_w8a8_packed_cooperative.wgsl +77 -0
  528. package/src/gpu/kernels/moe-grouped.js +132 -0
  529. package/src/gpu/kernels/moe-norm-router.js +91 -0
  530. package/src/gpu/kernels/moe-route.js +135 -0
  531. package/src/gpu/kernels/moe-shared-slot.js +219 -0
  532. package/src/gpu/kernels/moe_grouped_matmul_f32.wgsl +216 -0
  533. package/src/gpu/kernels/moe_grouped_matmul_ordered_f32.wgsl +158 -0
  534. package/src/gpu/kernels/moe_grouped_sort.wgsl +101 -0
  535. package/src/gpu/kernels/moe_norm_router_f32.wgsl +254 -0
  536. package/src/gpu/kernels/moe_route_gguf_split_f32.wgsl +117 -0
  537. package/src/gpu/kernels/moe_route_q4_0_f32.wgsl +81 -0
  538. package/src/gpu/kernels/moe_route_q4_f32.wgsl +250 -0
  539. package/src/gpu/kernels/moe_shared_slot_combine_f32.wgsl +91 -0
  540. package/src/gpu/kernels/moe_shared_slot_down_f32.wgsl +264 -0
  541. package/src/gpu/kernels/moe_shared_slot_down_pair_combine_f32.wgsl +254 -0
  542. package/src/gpu/kernels/moe_shared_slot_down_pair_combine_residual_f32.wgsl +262 -0
  543. package/src/gpu/kernels/moe_shared_slot_down_pair_f32.wgsl +187 -0
  544. package/src/gpu/kernels/moe_shared_slot_gu_f32.wgsl +264 -0
  545. package/src/gpu/kernels/moe_shared_slot_gu_pair_f32.wgsl +267 -0
  546. package/src/gpu/kernels/paired-activation.js +86 -0
  547. package/src/gpu/kernels/paired-attention.js +85 -0
  548. package/src/gpu/kernels/paired-cast.js +82 -0
  549. package/src/gpu/kernels/paired-linear-attention.js +180 -0
  550. package/src/gpu/kernels/paired-matmul.js +86 -0
  551. package/src/gpu/kernels/paired-rmsnorm.js +127 -0
  552. package/src/gpu/kernels/paired-rope.js +44 -0
  553. package/src/gpu/kernels/paired-split.js +74 -0
  554. package/src/gpu/kernels/paired_add.wgsl +261 -0
  555. package/src/gpu/kernels/paired_multiply.wgsl +261 -0
  556. package/src/gpu/kernels/paired_scale.wgsl +260 -0
  557. package/src/gpu/kernels/paired_scale_inplace.wgsl +259 -0
  558. package/src/gpu/kernels/paired_sigmoid.wgsl +266 -0
  559. package/src/gpu/kernels/paired_silu.wgsl +266 -0
  560. package/src/gpu/kernels/paired_silu_unary.wgsl +260 -0
  561. package/src/gpu/kernels/pipeline-cache.d.ts +37 -4
  562. package/src/gpu/kernels/pipeline-cache.js +185 -88
  563. package/src/gpu/kernels/ple.js +145 -0
  564. package/src/gpu/kernels/ple_conv.wgsl +104 -0
  565. package/src/gpu/kernels/ple_gate.wgsl +131 -0
  566. package/src/gpu/kernels/q4_0.js +200 -0
  567. package/src/gpu/kernels/qsa-indexer.d.ts +105 -0
  568. package/src/gpu/kernels/qsa-indexer.js +304 -0
  569. package/src/gpu/kernels/qsa_block_keys.wgsl +110 -0
  570. package/src/gpu/kernels/qsa_block_scores.wgsl +73 -0
  571. package/src/gpu/kernels/qsa_prepare.wgsl +116 -0
  572. package/src/gpu/kernels/qsa_select.wgsl +172 -0
  573. package/src/gpu/kernels/quantize_w8a8.wgsl +81 -0
  574. package/src/gpu/kernels/qwen_attention_split_q_gate.js +71 -0
  575. package/src/gpu/kernels/qwen_attention_split_q_gate.wgsl +26 -0
  576. package/src/gpu/kernels/qwen_linear_attention_decay_paired.wgsl +266 -0
  577. package/src/gpu/kernels/qwen_linear_attention_log_decay_paired.wgsl +273 -0
  578. package/src/gpu/kernels/qwen_linear_attention_prepare.js +153 -0
  579. package/src/gpu/kernels/qwen_linear_attention_prepare.wgsl +120 -0
  580. package/src/gpu/kernels/qwen_linear_attention_prepare_paired.wgsl +267 -0
  581. package/src/gpu/kernels/qwen_linear_attention_scale_paired.wgsl +276 -0
  582. package/src/gpu/kernels/relu.js +5 -12
  583. package/src/gpu/kernels/repeat_channels.js +22 -9
  584. package/src/gpu/kernels/repeat_channels.wgsl +12 -5
  585. package/src/gpu/kernels/repeat_channels_f16.wgsl +12 -5
  586. package/src/gpu/kernels/rescale_w8a8.wgsl +74 -0
  587. package/src/gpu/kernels/residual.js +5 -0
  588. package/src/gpu/kernels/rmsnorm-pair.js +2 -2
  589. package/src/gpu/kernels/rmsnorm-qk.js +27 -5
  590. package/src/gpu/kernels/rmsnorm-stats.js +62 -78
  591. package/src/gpu/kernels/rmsnorm.js +74 -96
  592. package/src/gpu/kernels/rmsnorm.wgsl +9 -3
  593. package/src/gpu/kernels/rmsnorm_f16.wgsl +2 -0
  594. package/src/gpu/kernels/rmsnorm_paired.wgsl +321 -0
  595. package/src/gpu/kernels/rmsnorm_paired_sliced.wgsl +326 -0
  596. package/src/gpu/kernels/rmsnorm_qk.wgsl +32 -8
  597. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +7 -7
  598. package/src/gpu/kernels/rope-dispatch.js +26 -0
  599. package/src/gpu/kernels/rope-precompute.js +29 -28
  600. package/src/gpu/kernels/rope-qk.js +35 -24
  601. package/src/gpu/kernels/rope.js +5 -2
  602. package/src/gpu/kernels/rope.wgsl +2 -2
  603. package/src/gpu/kernels/rope_precompute.wgsl +150 -6
  604. package/src/gpu/kernels/rope_precompute_paired.wgsl +285 -0
  605. package/src/gpu/kernels/rope_qk.wgsl +3 -2
  606. package/src/gpu/kernels/rope_qk_paired.wgsl +341 -0
  607. package/src/gpu/kernels/sample.js +115 -244
  608. package/src/gpu/kernels/sample.wgsl +53 -38
  609. package/src/gpu/kernels/sample_f16.wgsl +53 -38
  610. package/src/gpu/kernels/scale.js +6 -1
  611. package/src/gpu/kernels/scatter_add_routes_f32.wgsl +43 -0
  612. package/src/gpu/kernels/shader-cache.d.ts +7 -1
  613. package/src/gpu/kernels/shader-cache.js +61 -20
  614. package/src/gpu/kernels/shader-source-scope.d.ts +14 -0
  615. package/src/gpu/kernels/shader-source-scope.js +127 -12
  616. package/src/gpu/kernels/sigmoid-gated-add.js +68 -0
  617. package/src/gpu/kernels/sigmoid_gated_add_f32.wgsl +76 -0
  618. package/src/gpu/kernels/silu.js +56 -136
  619. package/src/gpu/kernels/silu.wgsl +54 -3
  620. package/src/gpu/kernels/silu_f16.wgsl +3 -2
  621. package/src/gpu/kernels/softmax.js +57 -19
  622. package/src/gpu/kernels/softmax_topk_logit_rank.wgsl +137 -0
  623. package/src/gpu/kernels/softmax_topk_ranked.wgsl +132 -0
  624. package/src/gpu/kernels/softmax_topk_strided.wgsl +80 -0
  625. package/src/gpu/kernels/speculative-accept.js +97 -0
  626. package/src/gpu/kernels/speculative_accept.wgsl +38 -0
  627. package/src/gpu/kernels/speculative_accept_gpu_limit.wgsl +42 -0
  628. package/src/gpu/kernels/split-dispatch.js +18 -0
  629. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +3 -6
  630. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +4 -11
  631. package/src/gpu/kernels/split_qg.js +6 -2
  632. package/src/gpu/kernels/split_qg.wgsl +3 -2
  633. package/src/gpu/kernels/split_qg_f16.wgsl +3 -2
  634. package/src/gpu/kernels/split_qg_paired.wgsl +59 -0
  635. package/src/gpu/kernels/split_qkv.js +7 -16
  636. package/src/gpu/kernels/split_qkv_paired.wgsl +75 -0
  637. package/src/gpu/kernels/subgroup-partition.js +11 -0
  638. package/src/gpu/kernels/uniform-encoding.js +20 -0
  639. package/src/gpu/kernels/uniform-utils.js +44 -49
  640. package/src/gpu/kernels/vision-average-pool.js +2 -5
  641. package/src/gpu/kernels/vision-patch-embed.js +2 -5
  642. package/src/gpu/kernels/vision-position-embedding.js +2 -5
  643. package/src/gpu/kernels/vision-rope-2d.js +2 -5
  644. package/src/gpu/kernels/vision-spatial-merge.js +2 -5
  645. package/src/gpu/kernels/w8a8.js +166 -0
  646. package/src/gpu/prepared-decode-workspace.d.ts +11 -0
  647. package/src/gpu/prepared-decode-workspace.js +104 -0
  648. package/src/gpu/recorder-timestamps.js +63 -0
  649. package/src/gpu/step-contract.d.ts +27 -0
  650. package/src/gpu/step-contract.js +26 -0
  651. package/src/gpu/submit-tracker.js +1 -17
  652. package/src/gpu/tensor.d.ts +13 -4
  653. package/src/gpu/tensor.js +38 -19
  654. package/src/gpu/uniform-cache.d.ts +8 -4
  655. package/src/gpu/uniform-cache.js +41 -25
  656. package/src/gpu/weight-buffer.d.ts +10 -1
  657. package/src/gpu/weight-buffer.js +2 -1
  658. package/src/index.d.ts +1 -0
  659. package/src/index.js +1 -0
  660. package/src/inference/choice-scoring.js +70 -0
  661. package/src/inference/decode-buffers.d.ts +11 -3
  662. package/src/inference/decode-buffers.js +93 -63
  663. package/src/inference/generation-output.js +128 -0
  664. package/src/inference/generation-step.d.ts +14 -0
  665. package/src/inference/generation-step.js +26 -0
  666. package/src/inference/kv-cache/base.d.ts +7 -2
  667. package/src/inference/kv-cache/base.js +10 -8
  668. package/src/inference/kv-cache/copy-state.js +39 -0
  669. package/src/inference/kv-cache/layer-range.js +76 -0
  670. package/src/inference/kv-cache/mixed-geometry.d.ts +3 -2
  671. package/src/inference/kv-cache/mixed-geometry.js +25 -1
  672. package/src/inference/kv-cache/quantized.d.ts +1 -1
  673. package/src/inference/kv-cache/quantized.js +1 -1
  674. package/src/inference/kv-cache/sliding-window.d.ts +2 -2
  675. package/src/inference/kv-cache/sliding-window.js +58 -183
  676. package/src/inference/kv-cache/tiered.d.ts +1 -1
  677. package/src/inference/kv-cache/tiered.js +1 -1
  678. package/src/inference/kv-cache/types.js +2 -28
  679. package/src/inference/kv-cache/update.js +104 -157
  680. package/src/inference/moe-router.d.ts +5 -0
  681. package/src/inference/moe-router.js +42 -31
  682. package/src/inference/pipelines/audio/index.js +5 -2
  683. package/src/inference/pipelines/audio/mel.js +116 -81
  684. package/src/inference/pipelines/context.js +109 -79
  685. package/src/inference/pipelines/factory.js +27 -5
  686. package/src/inference/pipelines/registry.js +9 -3
  687. package/src/inference/pipelines/shader-scoped-pipeline.d.ts +24 -0
  688. package/src/inference/pipelines/shader-scoped-pipeline.js +243 -11
  689. package/src/inference/pipelines/shared/clipped-linear.js +1 -1
  690. package/src/inference/pipelines/structured/json-head-pipeline.js +1 -1
  691. package/src/inference/pipelines/text/activity-weights.js +110 -0
  692. package/src/inference/pipelines/text/attention/cache-cast.js +15 -0
  693. package/src/inference/pipelines/text/attention/dispatch-params.js +2 -1
  694. package/src/inference/pipelines/text/attention/interpreter/immediate.js +16 -19
  695. package/src/inference/pipelines/text/attention/interpreter/recorded.js +20 -21
  696. package/src/inference/pipelines/text/attention/interpreter.js +34 -38
  697. package/src/inference/pipelines/text/attention/paired-contract.js +29 -0
  698. package/src/inference/pipelines/text/attention/plan.d.ts +2 -0
  699. package/src/inference/pipelines/text/attention/plan.js +11 -4
  700. package/src/inference/pipelines/text/attention/projection-executor.js +10 -27
  701. package/src/inference/pipelines/text/attention/projection-plan.js +24 -31
  702. package/src/inference/pipelines/text/attention/projections.js +10 -27
  703. package/src/inference/pipelines/text/attention/query-transform.js +2 -2
  704. package/src/inference/pipelines/text/attention/recorded-health.js +2 -1
  705. package/src/inference/pipelines/text/attention/rope-observation.js +26 -3
  706. package/src/inference/pipelines/text/chat-format.js +2 -1
  707. package/src/inference/pipelines/text/config/normalization.js +115 -226
  708. package/src/inference/pipelines/text/config/vision-normalizers.js +240 -0
  709. package/src/inference/pipelines/text/config-components.js +49 -0
  710. package/src/inference/pipelines/text/config.d.ts +45 -1
  711. package/src/inference/pipelines/text/config.js +26 -38
  712. package/src/inference/pipelines/text/debug-utils/config.js +13 -156
  713. package/src/inference/pipelines/text/debug-utils/health-metrics.js +8 -23
  714. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +5 -1
  715. package/src/inference/pipelines/text/embed.js +38 -4
  716. package/src/inference/pipelines/text/embedding-batch.js +23 -0
  717. package/src/inference/pipelines/text/embedding-extraction.js +7 -2
  718. package/src/inference/pipelines/text/embedding-normalization.js +2 -1
  719. package/src/inference/pipelines/text/execution-plan.d.ts +16 -0
  720. package/src/inference/pipelines/text/execution-plan.js +49 -2
  721. package/src/inference/pipelines/text/execution-runtime-builders.js +31 -0
  722. package/src/inference/pipelines/text/execution-v1/validation.js +3 -3
  723. package/src/inference/pipelines/text/execution-v1.js +14 -14
  724. package/src/inference/pipelines/text/execution.js +78 -128
  725. package/src/inference/pipelines/text/ffn/activation-observation.js +45 -0
  726. package/src/inference/pipelines/text/ffn/decode-intermediate.js +10 -0
  727. package/src/inference/pipelines/text/ffn/dense-executor.js +55 -50
  728. package/src/inference/pipelines/text/ffn/dense-plan.js +28 -27
  729. package/src/inference/pipelines/text/ffn/dense.js +28 -48
  730. package/src/inference/pipelines/text/ffn/moe.js +67 -3
  731. package/src/inference/pipelines/text/ffn/paired.js +144 -0
  732. package/src/inference/pipelines/text/ffn/qwen-shared-expert.js +212 -0
  733. package/src/inference/pipelines/text/ffn/sandwich.js +8 -3
  734. package/src/inference/pipelines/text/ffn/standard.js +67 -16
  735. package/src/inference/pipelines/text/finiteness-guard-status.js +9 -0
  736. package/src/inference/pipelines/text/generation-request.d.ts +36 -0
  737. package/src/inference/pipelines/text/generation-request.js +73 -0
  738. package/src/inference/pipelines/text/generator/activation-storage.js +15 -0
  739. package/src/inference/pipelines/text/generator/batch-retry-policy.js +20 -0
  740. package/src/inference/pipelines/text/generator/capsule-generation.js +37 -0
  741. package/src/inference/pipelines/text/generator/compiled-greedy-window.js +154 -0
  742. package/src/inference/pipelines/text/generator/decode-components.js +24 -0
  743. package/src/inference/pipelines/text/generator/decode-readback.js +73 -0
  744. package/src/inference/pipelines/text/generator/decode-runtime.js +396 -392
  745. package/src/inference/pipelines/text/generator/decode.js +593 -637
  746. package/src/inference/pipelines/text/generator/diagnostic-sequence.d.ts +63 -0
  747. package/src/inference/pipelines/text/generator/diagnostic-sequence.js +106 -0
  748. package/src/inference/pipelines/text/generator/diffusion.js +24 -12
  749. package/src/inference/pipelines/text/generator/greedy-head-policy.js +15 -0
  750. package/src/inference/pipelines/text/generator/logits-config.js +11 -4
  751. package/src/inference/pipelines/text/generator/per-layer-buffer.js +21 -0
  752. package/src/inference/pipelines/text/generator/prefill-observation.js +25 -1
  753. package/src/inference/pipelines/text/generator/prefill-policy.js +30 -18
  754. package/src/inference/pipelines/text/generator/prefill-runtime.js +511 -511
  755. package/src/inference/pipelines/text/generator/recorded-step-executor.js +70 -0
  756. package/src/inference/pipelines/text/generator/recorder-stats.js +17 -0
  757. package/src/inference/pipelines/text/generator/recovery.js +6 -0
  758. package/src/inference/pipelines/text/generator/session-context.js +19 -7
  759. package/src/inference/pipelines/text/generator/speculative-runtime.js +82 -0
  760. package/src/inference/pipelines/text/generator/text.js +17 -149
  761. package/src/inference/pipelines/text/generator/token-selection.d.ts +7 -0
  762. package/src/inference/pipelines/text/generator/token-selection.js +59 -0
  763. package/src/inference/pipelines/text/generator/training.js +39 -58
  764. package/src/inference/pipelines/text/generator/whole-token-lane.js +371 -0
  765. package/src/inference/pipelines/text/generator/whole-token-observation.d.ts +72 -0
  766. package/src/inference/pipelines/text/generator/whole-token-observation.js +502 -0
  767. package/src/inference/pipelines/text/generator/whole-token-program.d.ts +93 -0
  768. package/src/inference/pipelines/text/generator/whole-token-program.js +731 -0
  769. package/src/inference/pipelines/text/generator/whole-token-template.js +437 -0
  770. package/src/inference/pipelines/text/generator-runtime.d.ts +107 -0
  771. package/src/inference/pipelines/text/generator-runtime.js +24 -64
  772. package/src/inference/pipelines/text/generator-steps.js +49 -49
  773. package/src/inference/pipelines/text/generator.d.ts +10 -1
  774. package/src/inference/pipelines/text/generator.js +96 -166
  775. package/src/inference/pipelines/text/hyper-connection/decoder-layer.d.ts +75 -0
  776. package/src/inference/pipelines/text/hyper-connection/decoder-layer.js +132 -0
  777. package/src/inference/pipelines/text/hyper-connection/layer-executor.js +104 -0
  778. package/src/inference/pipelines/text/hyper-connection/logits-input.js +20 -0
  779. package/src/inference/pipelines/text/hyper-connection/projection.js +21 -0
  780. package/src/inference/pipelines/text/hyper-connection/sequence-transaction.js +145 -0
  781. package/src/inference/pipelines/text/hyper-connection/session.d.ts +39 -0
  782. package/src/inference/pipelines/text/hyper-connection/session.js +236 -0
  783. package/src/inference/pipelines/text/image-transcription.js +2 -3
  784. package/src/inference/pipelines/text/indexed-attention/attention-runner.js +49 -0
  785. package/src/inference/pipelines/text/indexed-attention/qsa-cache.d.ts +54 -0
  786. package/src/inference/pipelines/text/indexed-attention/qsa-cache.js +256 -0
  787. package/src/inference/pipelines/text/init-qkv-fusion.js +121 -104
  788. package/src/inference/pipelines/text/init-rope.d.ts +26 -0
  789. package/src/inference/pipelines/text/init-rope.js +109 -14
  790. package/src/inference/pipelines/text/init-tokenizer.js +29 -0
  791. package/src/inference/pipelines/text/init.d.ts +28 -10
  792. package/src/inference/pipelines/text/init.js +65 -74
  793. package/src/inference/pipelines/text/kernel-trace.js +33 -11
  794. package/src/inference/pipelines/text/layer-execution-contract.js +1 -1
  795. package/src/inference/pipelines/text/layer-input-execution.js +10 -9
  796. package/src/inference/pipelines/text/layer-partition-contract.d.ts +47 -0
  797. package/src/inference/pipelines/text/layer-partition-contract.js +220 -0
  798. package/src/inference/pipelines/text/layer-plan-gpu.js +153 -80
  799. package/src/inference/pipelines/text/layer-plan.js +9 -2
  800. package/src/inference/pipelines/text/layer.js +34 -47
  801. package/src/inference/pipelines/text/lifecycle.js +79 -178
  802. package/src/inference/pipelines/text/linear-attention/activity.js +29 -0
  803. package/src/inference/pipelines/text/linear-attention/norm-ab.js +59 -0
  804. package/src/inference/pipelines/text/linear-attention/plan.js +35 -34
  805. package/src/inference/pipelines/text/linear-attention/state-checkpoint.js +83 -0
  806. package/src/inference/pipelines/text/linear-attention/state-clone.js +109 -0
  807. package/src/inference/pipelines/text/linear-attention/state-layout.js +45 -0
  808. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +48 -45
  809. package/src/inference/pipelines/text/linear-attention.d.ts +20 -0
  810. package/src/inference/pipelines/text/linear-attention.js +130 -135
  811. package/src/inference/pipelines/text/load-errors.js +2 -19
  812. package/src/inference/pipelines/text/load-policy.js +55 -0
  813. package/src/inference/pipelines/text/logits/gpu-executor.js +130 -103
  814. package/src/inference/pipelines/text/logits/gpu.js +93 -70
  815. package/src/inference/pipelines/text/logits/index.d.ts +14 -2
  816. package/src/inference/pipelines/text/logits/index.js +420 -352
  817. package/src/inference/pipelines/text/logits/output-transform.js +28 -23
  818. package/src/inference/pipelines/text/logits/precision-policy.js +1 -1
  819. package/src/inference/pipelines/text/logits/types.d.ts +8 -1
  820. package/src/inference/pipelines/text/lora-apply.js +1 -1
  821. package/src/inference/pipelines/text/model-load.js +71 -117
  822. package/src/inference/pipelines/text/moe/gpu-executor.js +39 -17
  823. package/src/inference/pipelines/text/moe/grouped-prefill.js +67 -0
  824. package/src/inference/pipelines/text/moe/qwen-route-executor.d.ts +41 -0
  825. package/src/inference/pipelines/text/moe/qwen-route-executor.js +125 -0
  826. package/src/inference/pipelines/text/moe/routed-expert-chunks.js +95 -0
  827. package/src/inference/pipelines/text/moe-gpu.js +132 -50
  828. package/src/inference/pipelines/text/moe-impl.d.ts +23 -3
  829. package/src/inference/pipelines/text/moe-shape-validator.js +12 -0
  830. package/src/inference/pipelines/text/ngram/ngram-addresses.d.ts +30 -0
  831. package/src/inference/pipelines/text/ngram/ngram-addresses.js +134 -0
  832. package/src/inference/pipelines/text/ngram/ngram-table.d.ts +89 -0
  833. package/src/inference/pipelines/text/ngram/ngram-table.js +233 -0
  834. package/src/inference/pipelines/text/ngram/ple-layer.d.ts +54 -0
  835. package/src/inference/pipelines/text/ngram/ple-layer.js +115 -0
  836. package/src/inference/pipelines/text/operator-events.d.ts +129 -0
  837. package/src/inference/pipelines/text/operator-events.js +19 -2
  838. package/src/inference/pipelines/text/ops.js +40 -30
  839. package/src/inference/pipelines/text/paired-execution.js +140 -0
  840. package/src/inference/pipelines/text/partition-attempt.js +105 -0
  841. package/src/inference/pipelines/text/partition-execution.js +347 -0
  842. package/src/inference/pipelines/text/per-layer/materialize.js +3 -10
  843. package/src/inference/pipelines/text/per-layer/plan.js +3 -10
  844. package/src/inference/pipelines/text/per-layer-inputs.js +3 -10
  845. package/src/inference/pipelines/text/pipeline-load-timing.js +2 -1
  846. package/src/inference/pipelines/text/pipeline.d.ts +227 -0
  847. package/src/inference/pipelines/text/pipeline.js +629 -0
  848. package/src/inference/pipelines/text/precomputed-input-norm.js +29 -0
  849. package/src/inference/pipelines/text/probes.js +65 -23
  850. package/src/inference/pipelines/text/resident-partition-contract.d.ts +179 -0
  851. package/src/inference/pipelines/text/resident-partition-contract.js +98 -0
  852. package/src/inference/pipelines/text/resident-partition.js +403 -0
  853. package/src/inference/pipelines/text/resident-pipeline-preparation.js +341 -0
  854. package/src/inference/pipelines/text/sampling-config.d.ts +16 -0
  855. package/src/inference/pipelines/text/sampling-config.js +7 -4
  856. package/src/inference/pipelines/text/sequence-state.d.ts +68 -0
  857. package/src/inference/pipelines/text/sequence-state.js +514 -0
  858. package/src/inference/pipelines/text/stage-names.d.ts +88 -0
  859. package/src/inference/pipelines/text/state.d.ts +17 -8
  860. package/src/inference/pipelines/text/state.js +10 -0
  861. package/src/inference/pipelines/text/stopping.js +17 -0
  862. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +77 -0
  863. package/src/inference/pipelines/text/tsir-fixture-writer.js +3 -12
  864. package/src/inference/pipelines/text/types.d.ts +65 -111
  865. package/src/inference/pipelines/text/warm-generation.d.ts +27 -0
  866. package/src/inference/pipelines/text/warm-generation.js +34 -0
  867. package/src/inference/pipelines/text/weight-loading-config.js +16 -10
  868. package/src/inference/pipelines/text/weights.d.ts +103 -1
  869. package/src/inference/pipelines/text/weights.js +0 -17
  870. package/src/inference/pipelines/text.d.ts +3 -211
  871. package/src/inference/pipelines/text.js +16 -578
  872. package/src/inference/pipelines/video/feature-concat.js +30 -0
  873. package/src/inference/pipelines/video/frame-sampler.js +4 -0
  874. package/src/inference/pipelines/video/index.js +33 -44
  875. package/src/inference/pipelines/vision/encoder.js +1 -1
  876. package/src/inference/pipelines/vision/gemma4.js +6 -7
  877. package/src/inference/pipelines/vision/glmocr.js +11 -13
  878. package/src/inference/pipelines/vision/index.js +14 -3
  879. package/src/inference/pipelines/vision/support.js +93 -0
  880. package/src/inference/pipelines/vision/weight-loading.js +111 -0
  881. package/src/inference/rerank.d.ts +44 -0
  882. package/src/inference/resident-block-contract.d.ts +35 -0
  883. package/src/inference/resident-block-contract.js +105 -0
  884. package/src/inference/resident-output-state.d.ts +14 -0
  885. package/src/inference/resident-output-state.js +89 -0
  886. package/src/inference/resident-publication-contract.d.ts +32 -0
  887. package/src/inference/resident-publication-contract.js +104 -0
  888. package/src/inference/resident-state-contract.d.ts +26 -0
  889. package/src/inference/resident-state-contract.js +135 -0
  890. package/src/inference/resident-target-state.d.ts +32 -0
  891. package/src/inference/resident-target-state.js +152 -0
  892. package/src/inference/sampling-rng.js +35 -0
  893. package/src/inference/token-sampling.js +76 -0
  894. package/src/inference/tokenizer.d.ts +2 -0
  895. package/src/inference/tokenizer.js +16 -0
  896. package/src/inference/tokenizers/bundled/incremental-decoder.d.ts +16 -0
  897. package/src/inference/tokenizers/bundled/incremental-decoder.js +100 -0
  898. package/src/inference/tokenizers/bundled/normalizer.js +44 -0
  899. package/src/inference/tokenizers/bundled.js +25 -42
  900. package/src/inference/tokenizers/sentencepiece.js +4 -26
  901. package/src/loader/doppler-loader.d.ts +13 -0
  902. package/src/loader/doppler-loader.js +34 -62
  903. package/src/loader/dtype-utils.js +1 -19
  904. package/src/loader/embedding-limit-preflight.js +77 -8
  905. package/src/loader/embedding-loader.js +43 -114
  906. package/src/loader/experts/expert-cache.js +1 -4
  907. package/src/loader/experts/expert-loader.d.ts +6 -1
  908. package/src/loader/experts/expert-loader.js +90 -29
  909. package/src/loader/experts/expert-preparation.d.ts +16 -0
  910. package/src/loader/experts/expert-preparation.js +57 -0
  911. package/src/loader/experts/loader-bindings.js +53 -0
  912. package/src/loader/experts/qwen-paging.d.ts +19 -0
  913. package/src/loader/experts/qwen-paging.js +70 -0
  914. package/src/loader/experts/routed-expert-cache.d.ts +56 -0
  915. package/src/loader/experts/routed-expert-cache.js +372 -0
  916. package/src/loader/experts/routed-expert-plan.d.ts +20 -0
  917. package/src/loader/experts/routed-expert-plan.js +46 -0
  918. package/src/loader/final-weights-loader.d.ts +7 -1
  919. package/src/loader/final-weights-loader.js +17 -40
  920. package/src/loader/hyper-connection-loader.d.ts +25 -0
  921. package/src/loader/hyper-connection-loader.js +150 -0
  922. package/src/loader/layer-loader.d.ts +8 -1
  923. package/src/loader/layer-loader.js +67 -97
  924. package/src/loader/layer-tensor-aliases.js +77 -11
  925. package/src/loader/layer-weight-shape.js +13 -0
  926. package/src/loader/load-timing.js +2 -0
  927. package/src/loader/loader-types.d.ts +24 -1
  928. package/src/loader/model-executor.js +27 -365
  929. package/src/loader/model-load.d.ts +18 -0
  930. package/src/loader/model-load.js +432 -0
  931. package/src/loader/ngram-table-bindings.d.ts +11 -0
  932. package/src/loader/ngram-table-bindings.js +61 -0
  933. package/src/loader/per-layer-input-loader.js +5 -83
  934. package/src/loader/q4k-reference-weight.js +84 -0
  935. package/src/loader/shard-cache.js +3 -0
  936. package/src/loader/tensor-load-error.js +20 -1
  937. package/src/loader/tensors/bf16-loader.js +3 -29
  938. package/src/loader/tensors/buffer-write.js +26 -0
  939. package/src/loader/tensors/fp8-loader.js +28 -0
  940. package/src/loader/tensors/load-executor.js +124 -117
  941. package/src/loader/tensors/q4_0-loader.js +32 -0
  942. package/src/loader/tensors/range-backed-weights.js +34 -0
  943. package/src/loader/tensors/source-transform.js +1 -1
  944. package/src/loader/tensors/tensor-loader.js +22 -6
  945. package/src/loader/tensors/w8a8-loader.js +36 -0
  946. package/src/loader/weight-downcast.js +9 -13
  947. package/src/loader/weight-representation-plan.d.ts +28 -0
  948. package/src/loader/weight-representation-plan.js +146 -0
  949. package/src/loader/weights.d.ts +1 -1
  950. package/src/memory/buffer-pool.d.ts +24 -4
  951. package/src/memory/buffer-pool.js +124 -128
  952. package/src/memory/device-budget.d.ts +10 -0
  953. package/src/memory/device-budget.js +89 -0
  954. package/src/memory/device-residency.js +60 -0
  955. package/src/partitions.d.ts +21 -0
  956. package/src/partitions.js +10 -0
  957. package/src/rules/execution-rules-contract-check.js +73 -570
  958. package/src/rules/generated/rule-bundle.json +1 -4305
  959. package/src/rules/greedy-sampling.js +16 -0
  960. package/src/rules/inference/capability-transforms.rules.json +11 -1
  961. package/src/rules/layer-pattern-contract-check.js +49 -222
  962. package/src/rules/rule-matcher.d.ts +62 -0
  963. package/src/rules/rule-matcher.js +132 -1
  964. package/src/rules/rule-registry.d.ts +75 -0
  965. package/src/rules/rule-registry.js +64 -24
  966. package/src/storage/artifact-storage-context.d.ts +99 -0
  967. package/src/storage/artifact-storage-context.js +71 -114
  968. package/src/storage/backend-selection.js +3 -0
  969. package/src/storage/backends/idb-store.d.ts +54 -0
  970. package/src/storage/backends/idb-store.js +12 -0
  971. package/src/storage/backends/memory-store.d.ts +38 -0
  972. package/src/storage/backends/memory-store.js +15 -6
  973. package/src/storage/backends/opfs-store.d.ts +2 -0
  974. package/src/storage/backends/opfs-store.js +6 -0
  975. package/src/storage/blake3.d.ts +17 -0
  976. package/src/storage/blake3.js +153 -68
  977. package/src/storage/distribution-transport.js +626 -3
  978. package/src/storage/download/http-contract.js +199 -0
  979. package/src/storage/download/http-transport.js +824 -0
  980. package/src/storage/download/integrity.d.ts +10 -2
  981. package/src/storage/download/integrity.js +39 -2
  982. package/src/storage/download/transport.js +2 -2
  983. package/src/storage/downloader.js +404 -388
  984. package/src/storage/export.d.ts +1 -0
  985. package/src/storage/export.js +22 -70
  986. package/src/storage/model-cache.d.ts +4 -1
  987. package/src/storage/model-cache.js +131 -52
  988. package/src/storage/model-read-session.d.ts +13 -0
  989. package/src/storage/model-read-session.js +76 -0
  990. package/src/storage/node-file-access.js +108 -0
  991. package/src/storage/reports.js +1 -4
  992. package/src/storage/runtime-env.js +6 -0
  993. package/src/storage/shard-manager.d.ts +10 -0
  994. package/src/storage/shard-manager.js +644 -579
  995. package/src/storage/shards/index.js +16 -0
  996. package/src/storage/shards/integrity.js +35 -49
  997. package/src/storage/source-artifact-store.d.ts +59 -0
  998. package/src/storage/source-artifact-store.js +24 -37
  999. package/src/storage/source-storage-context.d.ts +53 -0
  1000. package/src/storage/source-storage-context.js +298 -0
  1001. package/src/storage/verified-piece-reader.d.ts +23 -0
  1002. package/src/storage/verified-piece-reader.js +134 -0
  1003. package/src/storage/verified-piece-storage.d.ts +7 -0
  1004. package/src/storage/verified-piece-storage.js +16 -0
  1005. package/src/tooling/boundary-evidence.js +1 -4
  1006. package/src/tooling/browser-command-runner.d.ts +1 -1
  1007. package/src/tooling/browser-command-runner.js +2 -2
  1008. package/src/tooling/browser-harness/benchmark-comparison.js +105 -0
  1009. package/src/tooling/browser-harness/benchmark-execution.js +865 -0
  1010. package/src/tooling/browser-harness/decode-diagnostics.js +201 -0
  1011. package/src/tooling/browser-harness/inference-execution.js +543 -0
  1012. package/src/tooling/browser-harness/model-initialization.js +230 -0
  1013. package/src/tooling/browser-harness/model-resolution.js +134 -0
  1014. package/src/tooling/browser-harness/report.js +338 -0
  1015. package/src/tooling/browser-harness/request-timing.js +50 -0
  1016. package/src/tooling/browser-harness/request.js +264 -0
  1017. package/src/tooling/browser-harness/runtime-config.d.ts +26 -0
  1018. package/src/tooling/browser-harness/runtime-config.js +256 -0
  1019. package/src/tooling/browser-harness/text-evidence.d.ts +81 -0
  1020. package/src/tooling/browser-harness/text-evidence.js +695 -0
  1021. package/src/tooling/browser-harness/text-execution.d.ts +37 -0
  1022. package/src/tooling/browser-harness/text-execution.js +721 -0
  1023. package/src/tooling/browser-harness/text-input.d.ts +45 -0
  1024. package/src/tooling/browser-harness/text-input.js +722 -0
  1025. package/src/tooling/browser-harness/timing-diagnostics.js +416 -0
  1026. package/src/tooling/browser-harness-text-helpers.d.ts +31 -0
  1027. package/src/tooling/browser-harness-text-helpers.js +31 -0
  1028. package/src/tooling/browser-harness.d.ts +245 -0
  1029. package/src/tooling/browser-harness.js +337 -0
  1030. package/src/tooling/command-api.d.ts +1 -0
  1031. package/src/tooling/diagnose-runner.js +2 -2
  1032. package/src/tooling/distribution-shard-transport.d.ts +6 -0
  1033. package/src/tooling/execution-cost-ledger.d.ts +2 -1
  1034. package/src/tooling/execution-cost-ledger.js +10 -7
  1035. package/src/tooling/hf-registry/catalog-io.js +7 -5
  1036. package/src/tooling/litert-package-runtime.js +1 -98
  1037. package/src/tooling/model-capsule-pieces.js +47 -0
  1038. package/src/tooling/model-capsule-rig.js +528 -0
  1039. package/src/tooling/node-browser/execution.d.ts +1 -1
  1040. package/src/tooling/node-browser/static-server.js +25 -2
  1041. package/src/tooling/node-browser/transport.js +0 -43
  1042. package/src/tooling/node-capsule-artifact-store.js +24 -0
  1043. package/src/tooling/node-command-execution.js +2 -2
  1044. package/src/tooling/node-command-runner.d.ts +1 -1
  1045. package/src/tooling/node-converter/input.js +36 -5
  1046. package/src/tooling/node-converter/output.js +75 -1
  1047. package/src/tooling/node-converter.js +33 -34
  1048. package/src/tooling/node-source-runtime.js +5 -5
  1049. package/src/tooling/portable-source-path.js +26 -0
  1050. package/src/tooling/production-release.js +3 -6
  1051. package/src/tooling/program-bundle/choice-scoring-reference.js +22 -0
  1052. package/src/tooling/program-bundle/materialize.js +4 -0
  1053. package/src/tooling/program-bundle/wgsl-closure.js +3 -9
  1054. package/src/tooling/program-bundle-host.js +26 -0
  1055. package/src/tooling/program-bundle.js +11 -6
  1056. package/src/tooling/reference-verify.js +5 -17
  1057. package/src/tooling/registered-variant-calibration-job.js +1 -4
  1058. package/src/tooling/registered-variant-calibration.js +1 -4
  1059. package/src/tooling/source-artifact-adapter.d.ts +1 -0
  1060. package/src/tooling/source-artifact-adapter.js +33 -1
  1061. package/src/tooling/source-intake.js +1 -4
  1062. package/src/tooling/source-runtime-bundle.d.ts +4 -85
  1063. package/src/tooling/source-runtime-bundle.js +3 -3
  1064. package/src/tooling/source-runtime-converter-config.js +41 -8
  1065. package/src/tooling/test-harness.d.ts +173 -0
  1066. package/src/tooling/test-harness.js +476 -0
  1067. package/src/tooling-exports/device.d.ts +1 -0
  1068. package/src/tooling-exports/device.js +1 -0
  1069. package/src/tooling-exports/evidence.d.ts +3 -0
  1070. package/src/tooling-exports/manifest.d.ts +1 -1
  1071. package/src/tooling-exports/manifest.js +1 -1
  1072. package/src/tooling-exports/runtime.d.ts +6 -1
  1073. package/src/tooling-exports/runtime.js +6 -1
  1074. package/src/tooling-exports/storage.d.ts +8 -0
  1075. package/src/tooling-exports/storage.js +8 -0
  1076. package/src/tooling-exports/structured.d.ts +2 -1
  1077. package/src/tooling-exports/structured.js +2 -1
  1078. package/src/tooling-exports.shared.d.ts +12 -9
  1079. package/src/tooling-exports.shared.js +6 -12
  1080. package/src/training-browser.d.ts +5 -0
  1081. package/src/training.d.ts +7 -0
  1082. package/src/training.js +1 -0
  1083. package/src/version.js +1 -1
  1084. package/tests/fixtures/linear-core-activation-accuracy.json +12 -0
  1085. package/tests/fixtures/linear-core-refined-accuracy.json +128 -0
  1086. package/tests/fixtures/q4k-ordered-accumulation.json +87 -0
  1087. package/tests/fixtures/rmsnorm-inverse-root.json +18 -0
  1088. package/tests/kernels/browser/kernel-suite.js +230 -0
  1089. package/tests/kernels/browser/test-page.js +4 -6
  1090. package/tests/kernels/harness/benchmark.js +11 -3
  1091. package/tests/kernels/reference/dequant.js +23 -3
  1092. package/tests/kernels/reference/index.js +1 -0
  1093. package/tools/convert-safetensors-node.js +7 -1
  1094. package/src/capsule.d.ts +0 -8
  1095. package/src/capsule.js +0 -4
  1096. package/src/cli/serve/capsule-handler.d.ts +0 -33
  1097. package/src/cli/serve/capsule-handler.js +0 -105
  1098. package/src/cli/serve/capsule-http.js +0 -102
  1099. package/src/client/runtime/index.js +0 -1
  1100. package/src/client/runtime/source-runtime.js +0 -910
  1101. package/src/client/runtime/target-selector.js +0 -2
  1102. package/src/config/backward-registry-loader.d.ts +0 -3
  1103. package/src/config/capsule-serve.d.ts +0 -9
  1104. package/src/config/capsule-serve.js +0 -28
  1105. package/src/config/capsule-serve.schema.json +0 -14
  1106. package/src/config/index.d.ts +0 -52
  1107. package/src/config/index.js +0 -22
  1108. package/src/config/kernels/kernel-ref-digests.js +0 -348
  1109. package/src/config/runtime/diagnostics/debug-logits.json +0 -54
  1110. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +0 -58
  1111. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +0 -58
  1112. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +0 -57
  1113. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +0 -33
  1114. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +0 -58
  1115. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +0 -58
  1116. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +0 -81
  1117. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +0 -213
  1118. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +0 -85
  1119. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +0 -81
  1120. package/src/config/runtime/experiments/verify/gemma3-verify.json +0 -37
  1121. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +0 -36
  1122. package/src/config/runtime/experiments/verify/lfm2-verify.json +0 -44
  1123. package/src/config/runtime/experiments/verify/translategemma-verify.json +0 -37
  1124. package/src/config/runtime/model/gemma2-debug.json +0 -78
  1125. package/src/config/runtime/model/gemma2-pipeline-debug.json +0 -67
  1126. package/src/config/runtime/model/gemma2-pipeline.json +0 -76
  1127. package/src/config/runtime/model/gemma3-layer-probe.json +0 -86
  1128. package/src/config/runtime/model/qwen3-5-layer-probe.json +0 -53
  1129. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +0 -109
  1130. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +0 -91
  1131. package/src/config/runtime/optimization-candidates/registry.json +0 -5
  1132. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +0 -27
  1133. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +0 -43
  1134. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +0 -40
  1135. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +0 -46
  1136. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +0 -44
  1137. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +0 -61
  1138. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +0 -39
  1139. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +0 -42
  1140. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +0 -20
  1141. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +0 -27
  1142. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +0 -30
  1143. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +0 -62
  1144. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +0 -43
  1145. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +0 -64
  1146. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +0 -33
  1147. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +0 -17
  1148. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +0 -30
  1149. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +0 -110
  1150. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +0 -21
  1151. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +0 -53
  1152. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +0 -19
  1153. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +0 -53
  1154. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +0 -53
  1155. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +0 -86
  1156. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +0 -33
  1157. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +0 -17
  1158. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +0 -19
  1159. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +0 -47
  1160. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +0 -58
  1161. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +0 -87
  1162. package/src/config/runtime/profiles/simulation.json +0 -29
  1163. package/src/config/runtime/profiles/trace-layers.json +0 -125
  1164. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +0 -23
  1165. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +0 -30
  1166. package/src/config/runtime/profiles/turboquant-contiguous.json +0 -30
  1167. package/src/config/runtime/profiles/turboquant.json +0 -37
  1168. package/src/config/runtime/profiles/verbose-trace.json +0 -37
  1169. package/src/config/runtime/tiers/gemma4-128gb.json +0 -80
  1170. package/src/config/runtime/tiers/gemma4-16gb.json +0 -83
  1171. package/src/config/runtime/tiers/gemma4-24gb.json +0 -80
  1172. package/src/config/runtime/tiers/gemma4-32gb.json +0 -80
  1173. package/src/config/transforms/capability-transform-resolver.js +0 -4
  1174. package/src/config/transforms/execution-graph/fusion.js +0 -436
  1175. package/src/config/transforms/execution-graph/validation.js +0 -414
  1176. package/src/config/transforms/execution-graph-transforms.js +0 -987
  1177. package/src/converter/shard-packer.d.ts +0 -138
  1178. package/src/converter/shard-packer.js +0 -425
  1179. package/src/experimental/adapters/adapter-manager.d.ts +0 -200
  1180. package/src/experimental/adapters/adapter-manager.js +0 -511
  1181. package/src/experimental/adapters/adapter-registry.d.ts +0 -192
  1182. package/src/experimental/adapters/adapter-registry.js +0 -477
  1183. package/src/experimental/adapters/artifact-contract.d.ts +0 -83
  1184. package/src/experimental/adapters/artifact-contract.js +0 -257
  1185. package/src/experimental/adapters/index.d.ts +0 -114
  1186. package/src/experimental/adapters/index.js +0 -54
  1187. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +0 -162
  1188. package/src/experimental/bridge/trainer-artifact-bridge.js +0 -595
  1189. package/src/experimental/browser/browser-converter.d.ts +0 -71
  1190. package/src/experimental/browser/browser-converter.js +0 -978
  1191. package/src/experimental/browser/file-picker.d.ts +0 -63
  1192. package/src/experimental/browser/file-picker.js +0 -281
  1193. package/src/experimental/browser/gguf-parser-browser.js +0 -17
  1194. package/src/experimental/browser/quantization.js +0 -328
  1195. package/src/experimental/browser/safetensors-parser-browser.js +0 -347
  1196. package/src/experimental/browser/tensor-source-download.js +0 -245
  1197. package/src/experimental/browser/tensor-source-file.d.ts +0 -26
  1198. package/src/experimental/browser/tensor-source-file.js +0 -53
  1199. package/src/experimental/browser/tensor-source-http.d.ts +0 -29
  1200. package/src/experimental/browser/tensor-source-http.js +0 -130
  1201. package/src/experimental/diffusion/image-regression.d.ts +0 -31
  1202. package/src/experimental/diffusion/image-regression.js +0 -107
  1203. package/src/experimental/diffusion/index.d.ts +0 -32
  1204. package/src/experimental/diffusion/index.js +0 -18
  1205. package/src/experimental/diffusion/pipeline.js +0 -727
  1206. package/src/experimental/diffusion/scheduler.js +0 -153
  1207. package/src/experimental/diffusion/text-encoder-gpu.js +0 -947
  1208. package/src/experimental/diffusion/vae.js +0 -596
  1209. package/src/experimental/distribution/descriptor-transport.d.ts +0 -117
  1210. package/src/experimental/distribution/descriptor-transport.js +0 -342
  1211. package/src/experimental/distribution/p2p-control-plane.d.ts +0 -52
  1212. package/src/experimental/distribution/p2p-observability.d.ts +0 -116
  1213. package/src/experimental/distribution/p2p-observability.js +0 -303
  1214. package/src/experimental/distribution/p2p-webrtc-browser.d.ts +0 -37
  1215. package/src/experimental/distribution/shard-delivery/transport.js +0 -831
  1216. package/src/experimental/distribution/shard-delivery.d.ts +0 -251
  1217. package/src/experimental/energy/index.d.ts +0 -2
  1218. package/src/experimental/energy/index.js +0 -2
  1219. package/src/experimental/energy/pipeline.js +0 -697
  1220. package/src/experimental/energy/quintel.d.ts +0 -107
  1221. package/src/experimental/energy/quintel.js +0 -218
  1222. package/src/experimental/orchestration/energy-row-head-pipeline.js +0 -491
  1223. package/src/experimental/orchestration/index.d.ts +0 -21
  1224. package/src/experimental/orchestration/index.js +0 -12
  1225. package/src/formats/quantization-constants.d.ts +0 -11
  1226. package/src/formats/quantization-constants.js +0 -11
  1227. package/src/formats/tokenizer/types.js +0 -22
  1228. package/src/gpu/kernel-selector.d.ts +0 -11
  1229. package/src/gpu/kernels/activation-static-qdq.d.ts +0 -22
  1230. package/src/gpu/kernels/attention.d.ts +0 -253
  1231. package/src/gpu/kernels/backward/adam.d.ts +0 -29
  1232. package/src/gpu/kernels/backward/attention_backward.d.ts +0 -35
  1233. package/src/gpu/kernels/backward/bias_add_backward.d.ts +0 -17
  1234. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +0 -24
  1235. package/src/gpu/kernels/backward/conv2d_backward.d.ts +0 -31
  1236. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +0 -23
  1237. package/src/gpu/kernels/backward/embed_backward.d.ts +0 -29
  1238. package/src/gpu/kernels/backward/executor.d.ts +0 -34
  1239. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +0 -39
  1240. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +0 -32
  1241. package/src/gpu/kernels/backward/gelu_backward.d.ts +0 -16
  1242. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +0 -24
  1243. package/src/gpu/kernels/backward/index.d.ts +0 -26
  1244. package/src/gpu/kernels/backward/layernorm_backward.d.ts +0 -22
  1245. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -31
  1246. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +0 -22
  1247. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +0 -24
  1248. package/src/gpu/kernels/backward/rope_backward.d.ts +0 -25
  1249. package/src/gpu/kernels/backward/scale_backward.d.ts +0 -16
  1250. package/src/gpu/kernels/backward/silu_backward.d.ts +0 -16
  1251. package/src/gpu/kernels/backward/softmax_backward.d.ts +0 -16
  1252. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +0 -21
  1253. package/src/gpu/kernels/backward/utils.js +0 -2
  1254. package/src/gpu/kernels/cast.d.ts +0 -67
  1255. package/src/gpu/kernels/clamp.d.ts +0 -22
  1256. package/src/gpu/kernels/conv2d.d.ts +0 -30
  1257. package/src/gpu/kernels/cross_entropy_loss.d.ts +0 -21
  1258. package/src/gpu/kernels/depthwise_conv1d.d.ts +0 -23
  1259. package/src/gpu/kernels/depthwise_conv2d.d.ts +0 -29
  1260. package/src/gpu/kernels/dequant.d.ts +0 -92
  1261. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +0 -19
  1262. package/src/gpu/kernels/energy.d.ts +0 -113
  1263. package/src/gpu/kernels/feature-check.d.ts +0 -42
  1264. package/src/gpu/kernels/fused_ffn.d.ts +0 -97
  1265. package/src/gpu/kernels/fused_matmul_residual.d.ts +0 -46
  1266. package/src/gpu/kernels/fused_matmul_rmsnorm.d.ts +0 -64
  1267. package/src/gpu/kernels/gather.d.ts +0 -144
  1268. package/src/gpu/kernels/gelu.d.ts +0 -37
  1269. package/src/gpu/kernels/gemma4-route-expert.d.ts +0 -35
  1270. package/src/gpu/kernels/grouped_pointwise_conv2d.d.ts +0 -27
  1271. package/src/gpu/kernels/groupnorm.d.ts +0 -27
  1272. package/src/gpu/kernels/index.d.ts +0 -517
  1273. package/src/gpu/kernels/index.js +0 -408
  1274. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +0 -9
  1275. package/src/gpu/kernels/kernel-prewarm.d.ts +0 -3
  1276. package/src/gpu/kernels/kv-cache-write.d.ts +0 -22
  1277. package/src/gpu/kernels/kv-quantize.d.ts +0 -37
  1278. package/src/gpu/kernels/layernorm.d.ts +0 -33
  1279. package/src/gpu/kernels/linear_attention.d.ts +0 -27
  1280. package/src/gpu/kernels/lm-head-argmax.d.ts +0 -28
  1281. package/src/gpu/kernels/lm-head-select-logits.d.ts +0 -23
  1282. package/src/gpu/kernels/logit-merge.d.ts +0 -86
  1283. package/src/gpu/kernels/logit-merge.js +0 -266
  1284. package/src/gpu/kernels/matmul.d.ts +0 -89
  1285. package/src/gpu/kernels/modulate.d.ts +0 -29
  1286. package/src/gpu/kernels/moe.d.ts +0 -113
  1287. package/src/gpu/kernels/pixel_shuffle.d.ts +0 -27
  1288. package/src/gpu/kernels/relu.d.ts +0 -18
  1289. package/src/gpu/kernels/repeat_channels.d.ts +0 -21
  1290. package/src/gpu/kernels/residual.d.ts +0 -65
  1291. package/src/gpu/kernels/rmsnorm-pair.d.ts +0 -72
  1292. package/src/gpu/kernels/rmsnorm-qk.d.ts +0 -41
  1293. package/src/gpu/kernels/rmsnorm-stats.d.ts +0 -29
  1294. package/src/gpu/kernels/rmsnorm.d.ts +0 -76
  1295. package/src/gpu/kernels/rope-qk.d.ts +0 -41
  1296. package/src/gpu/kernels/rope.d.ts +0 -48
  1297. package/src/gpu/kernels/sample.d.ts +0 -71
  1298. package/src/gpu/kernels/scale.d.ts +0 -40
  1299. package/src/gpu/kernels/silu.d.ts +0 -77
  1300. package/src/gpu/kernels/soft-embedding.d.ts +0 -28
  1301. package/src/gpu/kernels/softmax.d.ts +0 -57
  1302. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +0 -44
  1303. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +0 -59
  1304. package/src/gpu/kernels/split_qg.d.ts +0 -50
  1305. package/src/gpu/kernels/split_qkv.d.ts +0 -53
  1306. package/src/gpu/kernels/transpose.d.ts +0 -21
  1307. package/src/gpu/kernels/types.d.ts +0 -21
  1308. package/src/gpu/kernels/upsample2d.d.ts +0 -21
  1309. package/src/gpu/kernels/utils.js +0 -9
  1310. package/src/gpu/kernels/vision-average-pool.d.ts +0 -13
  1311. package/src/gpu/kernels/vision-patch-embed.d.ts +0 -17
  1312. package/src/gpu/kernels/vision-position-embedding.d.ts +0 -13
  1313. package/src/gpu/kernels/vision-rope-2d.d.ts +0 -16
  1314. package/src/gpu/kernels/vision-spatial-merge.d.ts +0 -19
  1315. package/src/gpu/multi-model-recorder.d.ts +0 -21
  1316. package/src/gpu/multi-model-recorder.js +0 -31
  1317. package/src/gpu/partitioned-buffer-pool.d.ts +0 -28
  1318. package/src/gpu/partitioned-buffer-pool.js +0 -57
  1319. package/src/gpu/perf-guards.js +0 -71
  1320. package/src/inference/browser-harness/benchmark-execution.js +0 -872
  1321. package/src/inference/browser-harness/decode-diagnostics.js +0 -241
  1322. package/src/inference/browser-harness/inference-execution.js +0 -543
  1323. package/src/inference/browser-harness/model-initialization.js +0 -225
  1324. package/src/inference/browser-harness/model-resolution.js +0 -137
  1325. package/src/inference/browser-harness/report.js +0 -338
  1326. package/src/inference/browser-harness/request.js +0 -263
  1327. package/src/inference/browser-harness/runtime-config.d.ts +0 -25
  1328. package/src/inference/browser-harness/runtime-config.js +0 -255
  1329. package/src/inference/browser-harness/runtime-isolation.d.ts +0 -13
  1330. package/src/inference/browser-harness/runtime-report.d.ts +0 -26
  1331. package/src/inference/browser-harness/text-evidence.js +0 -694
  1332. package/src/inference/browser-harness/text-execution.js +0 -757
  1333. package/src/inference/browser-harness/text-input.js +0 -725
  1334. package/src/inference/browser-harness/timing-diagnostics.js +0 -414
  1335. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -3
  1336. package/src/inference/browser-harness-runtime-helpers.js +0 -3
  1337. package/src/inference/browser-harness-text-helpers.js +0 -31
  1338. package/src/inference/browser-harness.d.ts +0 -245
  1339. package/src/inference/browser-harness.js +0 -337
  1340. package/src/inference/expert-router.d.ts +0 -27
  1341. package/src/inference/expert-router.js +0 -55
  1342. package/src/inference/kv-cache/index.d.ts +0 -38
  1343. package/src/inference/kv-cache/index.js +0 -22
  1344. package/src/inference/kv-cache.d.ts +0 -39
  1345. package/src/inference/kv-cache.js +0 -20
  1346. package/src/inference/multi-model-network.d.ts +0 -139
  1347. package/src/inference/multi-model-network.js +0 -435
  1348. package/src/inference/multi-pipeline-pool.d.ts +0 -62
  1349. package/src/inference/multi-pipeline-pool.js +0 -161
  1350. package/src/inference/network-evolution.d.ts +0 -43
  1351. package/src/inference/pipelines/diffusion/init.d.ts +0 -24
  1352. package/src/inference/pipelines/diffusion/init.js +0 -138
  1353. package/src/inference/pipelines/diffusion/profile-timing.js +0 -10
  1354. package/src/inference/pipelines/diffusion/runtime-resources.js +0 -54
  1355. package/src/inference/pipelines/diffusion/sd3/execution.js +0 -805
  1356. package/src/inference/pipelines/diffusion/sd3/plan.js +0 -125
  1357. package/src/inference/pipelines/diffusion/sd3-transformer.js +0 -343
  1358. package/src/inference/pipelines/diffusion/sd3-weights.js +0 -287
  1359. package/src/inference/pipelines/diffusion/text-encoder.js +0 -195
  1360. package/src/inference/pipelines/diffusion/vae/execution.js +0 -803
  1361. package/src/inference/pipelines/diffusion/weight-contract.js +0 -53
  1362. package/src/inference/pipelines/diffusion/weights.js +0 -155
  1363. package/src/inference/pipelines/diffusion-gemma/config.d.ts +0 -27
  1364. package/src/inference/pipelines/diffusion-gemma/config.js +0 -149
  1365. package/src/inference/pipelines/diffusion-gemma/index.d.ts +0 -31
  1366. package/src/inference/pipelines/diffusion-gemma/index.js +0 -12
  1367. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +0 -69
  1368. package/src/inference/pipelines/diffusion-gemma/pipeline.js +0 -380
  1369. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +0 -128
  1370. package/src/inference/pipelines/diffusion-gemma/sampling.js +0 -395
  1371. package/src/inference/pipelines/rng.js +0 -17
  1372. package/src/inference/pipelines/structured/index.d.ts +0 -10
  1373. package/src/inference/pipelines/structured/index.js +0 -10
  1374. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +0 -8
  1375. package/src/inference/pipelines/text/attention/index.js +0 -24
  1376. package/src/inference/pipelines/text/attention/record.js +0 -2
  1377. package/src/inference/pipelines/text/attention/run.js +0 -2
  1378. package/src/inference/pipelines/text/config/validation.js +0 -9
  1379. package/src/inference/pipelines/text/debug-utils/index.js +0 -39
  1380. package/src/inference/pipelines/text/debug-utils/profiles.js +0 -8
  1381. package/src/inference/pipelines/text/debug-utils.js +0 -34
  1382. package/src/inference/pipelines/text/ffn/index.js +0 -16
  1383. package/src/inference/pipelines/text/model-load.d.ts +0 -59
  1384. package/src/inference/test-harness.d.ts +0 -172
  1385. package/src/inference/test-harness.js +0 -470
  1386. package/src/loader/dtype-utils.d.ts +0 -40
  1387. package/src/loader/embedding-loader.d.ts +0 -66
  1388. package/src/loader/index.d.ts +0 -152
  1389. package/src/loader/index.js +0 -126
  1390. package/src/loader/loader-state.d.ts +0 -51
  1391. package/src/loader/loader-state.js +0 -142
  1392. package/src/loader/manifest-config.d.ts +0 -118
  1393. package/src/loader/multi-model-loader.d.ts +0 -59
  1394. package/src/loader/multi-model-loader.js +0 -141
  1395. package/src/loader/quantization-constants.d.ts +0 -11
  1396. package/src/loader/quantization-constants.js +0 -13
  1397. package/src/loader/shard-resolver.d.ts +0 -13
  1398. package/src/loader/tensors/tensor-loader.d.ts +0 -197
  1399. package/src/loader/tensors/tensor-reader.d.ts +0 -33
  1400. package/src/loader/tensors/tensor-role.d.ts +0 -7
  1401. package/src/loader/weight-downcast.d.ts +0 -62
  1402. package/src/loader/weights.js +0 -4
  1403. package/src/loaders/index.d.ts +0 -4
  1404. package/src/loaders/index.js +0 -4
  1405. package/src/models/diffusiongemma.d.ts +0 -7
  1406. package/src/models/diffusiongemma.js +0 -25
  1407. package/src/models/embeddinggemma.d.ts +0 -7
  1408. package/src/models/embeddinggemma.js +0 -24
  1409. package/src/models/family.d.ts +0 -24
  1410. package/src/models/family.js +0 -25
  1411. package/src/models/gemma3.d.ts +0 -7
  1412. package/src/models/gemma3.js +0 -33
  1413. package/src/models/gemma4.d.ts +0 -7
  1414. package/src/models/gemma4.js +0 -65
  1415. package/src/models/qwen3.d.ts +0 -7
  1416. package/src/models/qwen3.js +0 -36
  1417. package/src/rules/capability-transform-resolver.js +0 -80
  1418. package/src/tooling/capsule-v2.d.ts +0 -11
  1419. package/src/tooling/model-capsule-forge.js +0 -478
  1420. package/src/tooling/source-runtime/bundle-materialize.js +0 -298
  1421. package/src/tooling-experimental-exports.browser.js +0 -2
  1422. package/src/tooling-experimental-exports.d.ts +0 -7
  1423. package/src/tooling-experimental-exports.js +0 -2
  1424. package/src/tooling-experimental-exports.shared.d.ts +0 -61
  1425. package/src/tooling-experimental-exports.shared.js +0 -57
  1426. /package/src/{config/transforms → converter}/execution-graph/dtype.js +0 -0
  1427. /package/src/{tooling/source-runtime/bundle-plan.js → formats/source-runtime.js} +0 -0
  1428. /package/src/{inference → tooling}/browser-harness/benchmark-rerank.js +0 -0
  1429. /package/src/{inference → tooling}/browser-harness/diffusion-performance.js +0 -0
  1430. /package/src/{inference → tooling}/browser-harness/runtime-isolation.js +0 -0
  1431. /package/src/{inference → tooling}/browser-harness/runtime-report.js +0 -0
  1432. /package/src/{inference → tooling}/browser-harness/suite-summary.js +0 -0
  1433. /package/src/{inference → tooling}/browser-harness-contract-helpers.js +0 -0
  1434. /package/src/{inference → tooling}/browser-harness-diffusion-energy-suites.js +0 -0
  1435. /package/src/{inference → tooling}/browser-harness-report-helpers.js +0 -0
@@ -0,0 +1,927 @@
1
+ import { widenAttentionKvToF32 } from './execution-graph/kv-storage.js';
2
+ export { widenAttentionKvToF32 } from './execution-graph/kv-storage.js';
3
+ import { F16_TO_F32_ACTIVATION_MAP, F32_TO_F16_ACTIVATION_MAP, KERNEL_FILE_PRECISION_PATCHES, cloneGraph, deriveKernelEntry, deriveKernelKey, deriveLinearDecodeF16KernelEntry, narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';
4
+ import { LAYER_PROJECTION_OPS, useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode, useGemma4TextF16ActivationsForLane } from './execution-graph/fusion.js';
5
+ import { ATTENTION_F16KV_TO_F32_ENTRIES, deriveMappedKernelEntry, deriveQ4DecodeF32ActivationKernelEntry, deriveQ4PrefillF32ActivationKernelEntry, findKernelKeyByFileAndEntry, hasKernelFile, remapDenseQ4KPrefillToQ4Native, remapStepKeys, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
6
+ export { remapDenseQ4KPrefillToQ4Native, removeSubgroups, widenProjectionWeightsToF32, widenToF32Activations } from './execution-graph/validation.js';
7
+ export { useGemma4Int4PleAf16Activations, useGemma4Int4PleSelectiveF16Decode } from './execution-graph/fusion.js';
8
+ export { narrowToF16Activations, useQwen36F16Activations, useQwenDecodeF16Matmuls, useQwenF16PrimaryMatmuls } from './execution-graph/dtype.js';// =============================================================================
9
+ // Execution Graph Transforms
10
+ // =============================================================================
11
+ //
12
+ // Pure functions that take an execution-v1 graph (as stamped in the manifest)
13
+ // and return a modified copy. Replaces the kernel path registry system.
14
+ //
15
+ // Each transform: (graph, ctx) => newGraph | null
16
+ // Returns null if not applicable (no-op).
17
+ // Must be pure — no mutation, return new objects.
18
+
19
+ // =============================================================================
20
+ // Helpers
21
+ // =============================================================================
22
+
23
+ /*
24
+ * Deep-clone an execution graph.
25
+ */
26
+
27
+ /*
28
+ * Shader files that require subgroups even though "subgroup" is not in the filename.
29
+ * Online attention kernels use subgroup reductions internally.
30
+ */
31
+
32
+ export function getKernelFilePrecisionPatch(kernel) {
33
+ return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
34
+ }
35
+
36
+ /*
37
+ * Find the first kernel key used by matching ops in a phase whose shader file
38
+ * satisfies the provided predicate.
39
+ */
40
+
41
+ /*
42
+ * Find an existing kernel key by shader file and entry point.
43
+ */
44
+
45
+ function normalizeLayerType(layerType) {
46
+ return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
47
+ }
48
+
49
+ function isLinearAttentionLayerType(layerType) {
50
+ const normalized = normalizeLayerType(layerType);
51
+ return normalized === 'linear_attention'
52
+ || normalized === 'linear'
53
+ || normalized === 'gated_delta'
54
+ || normalized === 'gated_delta_net';
55
+ }
56
+
57
+ function buildGroupedLayerEntries(baseStep, targetLayers, replacementKernelKey) {
58
+ const groupedEntries = [];
59
+ if (!Array.isArray(baseStep) || baseStep.length < 2) {
60
+ return groupedEntries;
61
+ }
62
+
63
+ const totalLayers = targetLayers.allLayers;
64
+ const targeted = targetLayers.matchingLayers;
65
+ const remaining = totalLayers.filter((layerIdx) => !targeted.includes(layerIdx));
66
+
67
+ if (remaining.length > 0) {
68
+ groupedEntries.push({
69
+ layers: remaining,
70
+ steps: [baseStep],
71
+ });
72
+ }
73
+ if (targeted.length > 0) {
74
+ const replacement = [...baseStep];
75
+ replacement[1] = replacementKernelKey;
76
+ groupedEntries.push({
77
+ layers: targeted,
78
+ steps: [replacement],
79
+ });
80
+ }
81
+
82
+ return groupedEntries;
83
+ }
84
+
85
+ // =============================================================================
86
+ // Transform: removeSubgroups
87
+ // =============================================================================
88
+
89
+ /*
90
+ * Remove subgroup shader dependencies from decode and postLayer steps.
91
+ * Prefill steps are left untouched (they already use tiled matmul).
92
+ *
93
+ * Returns null if the graph has no subgroup kernels.
94
+ *
95
+ */
96
+
97
+ // =============================================================================
98
+ // Transform: widenToF32Activations
99
+ // =============================================================================
100
+
101
+ /*
102
+ * Activation-only widening: f16-activation shaders → f32-activation variants
103
+ * that still use f16 for weights and KV cache. Requires shader-f16 for weight
104
+ * and KV buffer reads.
105
+ */
106
+
107
+ export function resolveF16ToF32ActivationKernel(kernel) {
108
+ return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
109
+ }
110
+
111
+ /*
112
+ * Activation-only narrowing: f32-activation shaders that still consume f16
113
+ * weights/KV are rewritten onto the matching f16-activation lane.
114
+ *
115
+ * This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
116
+ * runtime session explicitly requests f16 activations for an execution-v1
117
+ * graph that was authored with conservative f32 activation defaults.
118
+ */
119
+
120
+ F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
121
+ F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
122
+ // head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
123
+ // f16 requests fail-closed until that path has its own evidence.
124
+ F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
125
+
126
+ /*
127
+ * Correctness fallback: preserve f16 weights where possible, but widen both
128
+ * activations and KV-cache interactions onto the stable f32 execution lane.
129
+ * Used for alternate-plan recovery after finiteness failure.
130
+ */
131
+ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
132
+ ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
133
+ ['rope_f16.wgsl', 'rope.wgsl'],
134
+ ['residual_f16.wgsl', 'residual.wgsl'],
135
+ ['gelu_f16.wgsl', 'gelu.wgsl'],
136
+ ['silu_f16.wgsl', 'silu.wgsl'],
137
+ ['sample_f16.wgsl', 'sample.wgsl'],
138
+ ['gather_f16.wgsl', 'gather.wgsl'],
139
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
140
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
141
+ ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
142
+ ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
143
+ ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
144
+ ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
145
+ ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
146
+ ['attention_small_f16.wgsl', 'attention_small.wgsl'],
147
+ ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
148
+ ...ATTENTION_F16KV_TO_F32_ENTRIES,
149
+ ]);
150
+
151
+ /*
152
+ * Full f32 widening: every shader that uses `enable f16;` is replaced with a
153
+ * pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
154
+ * Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
155
+ */
156
+
157
+ /*
158
+ * Widen all f16-activation shaders to f32-activation equivalents.
159
+ *
160
+ * Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
161
+ * equivalent exists) or if no f16 activation shaders are present.
162
+ *
163
+ * NOTE: The caller is responsible for also updating session.activationDtype
164
+ * to reflect the widened dtype.
165
+ *
166
+ */
167
+
168
+ /*
169
+ * Widen an f16 execution graph onto the stable f32 correctness lane used for
170
+ * alternate-plan recovery after finiteness failure.
171
+ *
172
+ */
173
+ export function widenToF32CorrectnessFallback(graph, ctx) {
174
+ if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
175
+ return null;
176
+ }
177
+
178
+ const hasTargetShader = Object.values(graph.kernels).some(
179
+ (entry) => F16_TO_F32_CORRECTNESS_FALLBACK_MAP.has(entry.kernel)
180
+ );
181
+ if (!hasTargetShader) {
182
+ return null;
183
+ }
184
+
185
+ const result = cloneGraph(graph);
186
+ for (const [key, entry] of Object.entries(result.kernels)) {
187
+ const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
188
+ ?? deriveQ4PrefillF32ActivationKernelEntry(entry);
189
+ if (q4FallbackEntry) {
190
+ result.kernels[key] = q4FallbackEntry;
191
+ continue;
192
+ }
193
+ const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
194
+ if (replacement !== undefined) {
195
+ result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
196
+ }
197
+ }
198
+ return result;
199
+ }
200
+
201
+ /*
202
+ * Narrow f32-activation shaders back onto their f16-activation equivalents.
203
+ *
204
+ * Returns null if the graph has no supported f32-activation kernels to swap or
205
+ * if the runtime did not explicitly request f16 activations on an f16-capable
206
+ * GPU.
207
+ *
208
+ */
209
+
210
+ // =============================================================================
211
+ // Transform: swapPrefillAttention
212
+ // =============================================================================
213
+
214
+ const PREFILL_ATTENTION_PAIRS = new Map([
215
+ ['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
216
+ ['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
217
+ ['attention_streaming_f16.wgsl', 'attention_small_f16.wgsl'],
218
+ ['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
219
+ ]);
220
+
221
+ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
222
+ for (const entry of graph.prefill || []) {
223
+ if (Array.isArray(entry)) {
224
+ if (entry[1] === kernelKey) {
225
+ return true;
226
+ }
227
+ continue;
228
+ }
229
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
230
+ continue;
231
+ }
232
+ for (const step of entry.steps) {
233
+ if (Array.isArray(step) && step[1] === kernelKey) {
234
+ return true;
235
+ }
236
+ }
237
+ }
238
+ return false;
239
+ }
240
+
241
+ /*
242
+ * Swap prefill attention kernel between streaming and small variants.
243
+ *
244
+ * The `opts` parameter specifies the direction:
245
+ * { from: 'attention_streaming_f16kv.wgsl', to: 'attention_small_f16kv.wgsl' }
246
+ *
247
+ * If `from`/`to` are not provided, uses the bidirectional pair map.
248
+ * Returns null if no matching prefill attention kernel is found.
249
+ *
250
+ */
251
+ export function swapPrefillAttention(graph, ctx, opts) {
252
+ const from = opts?.from;
253
+ const to = opts?.to;
254
+
255
+ const result = cloneGraph(graph);
256
+ let changed = false;
257
+
258
+ for (const [key, entry] of Object.entries(result.kernels)) {
259
+ let target;
260
+
261
+ if (from && to) {
262
+ // Explicit direction: only swap if the kernel matches `from`
263
+ if (entry.kernel === from) {
264
+ target = to;
265
+ }
266
+ } else {
267
+ // Bidirectional: use pair map
268
+ target = PREFILL_ATTENTION_PAIRS.get(entry.kernel);
269
+ }
270
+
271
+ if (target !== undefined) {
272
+ const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
273
+ if (usedInPrefill) {
274
+ result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
275
+ changed = true;
276
+ }
277
+ }
278
+ }
279
+
280
+ return changed ? result : null;
281
+ }
282
+
283
+ // =============================================================================
284
+ // Transform: useHead256PrefillAttention
285
+ // =============================================================================
286
+
287
+ /*
288
+ * Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
289
+ *
290
+ */
291
+ export function useHead256SmallPrefillAttention(graph, ctx) {
292
+ return swapPrefillAttention(graph, ctx, {
293
+ from: 'attention_small_f16kv.wgsl',
294
+ to: 'attention_head256_f16kv.wgsl',
295
+ });
296
+ }
297
+
298
+ /*
299
+ * Promote prefill attention onto the fixed 256-dim shared-block kernel.
300
+ *
301
+ */
302
+ export function useHead256PrefillAttention(graph, ctx) {
303
+ let current = graph;
304
+ let changed = false;
305
+
306
+ const smallRemap = useHead256SmallPrefillAttention(current, ctx);
307
+ if (smallRemap) {
308
+ current = smallRemap;
309
+ changed = true;
310
+ }
311
+
312
+ const streamingRemap = swapPrefillAttention(current, ctx, {
313
+ from: 'attention_streaming_f16kv.wgsl',
314
+ to: 'attention_head256_f16kv.wgsl',
315
+ });
316
+ if (streamingRemap) {
317
+ current = streamingRemap;
318
+ changed = true;
319
+ }
320
+
321
+ return changed ? current : null;
322
+ }
323
+
324
+ // =============================================================================
325
+ // Transform: widenProjectionWeightsToF32
326
+ // =============================================================================
327
+
328
+ /*
329
+ * Known layer projection ops. Only these are widened; lm_head and embed are
330
+ * excluded.
331
+ */
332
+
333
+ function resolveDensePrefillProjectionKernel(ctx) {
334
+ return ctx.activationDtype === 'f16'
335
+ ? 'matmul_f16.wgsl'
336
+ : 'matmul_f16w_f32a.wgsl';
337
+ }
338
+
339
+ /*
340
+ * Replace projection matmul kernels with f32 weight variants.
341
+ *
342
+ * Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
343
+ * or embed.
344
+ *
345
+ * Returns null if no applicable projection kernels are found.
346
+ *
347
+ */
348
+
349
+ // =============================================================================
350
+ // Transform: remapDenseQ4KPrefillToQ4Native
351
+ // =============================================================================
352
+
353
+ /*
354
+ * Replace dense prefill projection kernels with Q4-native prefill variants.
355
+ *
356
+ * This applies only when the graph already exposes a compatible fused Q4 decode
357
+ * projection kernel. All prefill layer projections are remapped to the shared-A
358
+ * batched multicol Q4 prefill kernel so the transformed path remains valid for
359
+ * `M > 1` prefill workloads.
360
+ *
361
+ * Returns null when the graph does not have the required dense-prefill + Q4
362
+ * decode shape.
363
+ *
364
+ */
365
+
366
+ // =============================================================================
367
+ // Transform: remapQ4KPrefillToDense
368
+ // =============================================================================
369
+
370
+ /*
371
+ * Replace fused Q4K prefill projection kernels with dense tiled variants.
372
+ *
373
+ * Decode remains unchanged so the runtime can keep using fused Q4K decode while
374
+ * the loader exposes mixed dense+Q4K materializations for prefill.
375
+ *
376
+ * Returns null when the graph has no fused Q4K prefill projection kernels.
377
+ *
378
+ */
379
+ export function remapQ4KPrefillToDense(graph, ctx) {
380
+ const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
381
+ if (!LAYER_PROJECTION_OPS.has(step[0])) {
382
+ return false;
383
+ }
384
+ const entry = graph.kernels[step[1]];
385
+ return entry != null && entry.kernel.startsWith('fused_matmul_q4');
386
+ });
387
+ if (q4PrefillProjectionSteps.length === 0) {
388
+ return null;
389
+ }
390
+
391
+ const denseKernelFile = resolveDensePrefillProjectionKernel(ctx);
392
+ const result = cloneGraph(graph);
393
+ let denseKey = findKernelKeyByFileAndEntry(result, denseKernelFile, 'main');
394
+ if (!denseKey) {
395
+ const sourceKey = q4PrefillProjectionSteps[0][1];
396
+ const sourceEntry = result.kernels[sourceKey];
397
+ denseKey = deriveKernelKey(result.kernels, sourceKey, '_prefill_dense');
398
+ result.kernels[denseKey] = deriveKernelEntry(
399
+ sourceEntry,
400
+ denseKernelFile,
401
+ 'main',
402
+ null
403
+ );
404
+ }
405
+
406
+ let changed = false;
407
+ result.prefill = (result.prefill || []).map((step) => {
408
+ if (!LAYER_PROJECTION_OPS.has(step[0])) {
409
+ return step;
410
+ }
411
+ const entry = result.kernels[step[1]];
412
+ if (!entry || !entry.kernel.startsWith('fused_matmul_q4')) {
413
+ return step;
414
+ }
415
+ if (step[1] === denseKey) {
416
+ return step;
417
+ }
418
+ changed = true;
419
+ const next = [...step];
420
+ next[1] = denseKey;
421
+ return next;
422
+ });
423
+
424
+ return changed ? result : null;
425
+ }
426
+
427
+ // =============================================================================
428
+ // Transform: useLinearDecodeProjectionF16
429
+ // =============================================================================
430
+
431
+ /*
432
+ * Remap the linear-attention q_proj decode step onto the f16-activation fused
433
+ * Q4 kernel for linear-attention layers only. Full-attention layers keep the
434
+ * manifest-wide f32 activation contract.
435
+ *
436
+ * Only q_proj is remapped. o_proj is intentionally excluded: the o_proj
437
+ * output enters the residual stream directly, and f16 truncation there
438
+ * accumulates across the 18 linear-attention layers in the Qwen 3.5 pattern,
439
+ * corrupting the logit distribution (empirically verified: degenerate
440
+ * repetitive output under greedy decode). q_proj f16 is safe because the
441
+ * linear attention core absorbs the f16 input into its f32 internal state.
442
+ *
443
+ */
444
+ export function useLinearDecodeProjectionF16(graph, ctx) {
445
+ const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
446
+ if (!layerTypes || layerTypes.length === 0) {
447
+ return null;
448
+ }
449
+
450
+ const matchingLayers = layerTypes
451
+ .map((layerType, layerIdx) => ({ layerType, layerIdx }))
452
+ .filter(({ layerType }) => isLinearAttentionLayerType(layerType))
453
+ .map(({ layerIdx }) => layerIdx);
454
+ if (matchingLayers.length === 0) {
455
+ return null;
456
+ }
457
+
458
+ const result = cloneGraph(graph);
459
+ const targetLayers = {
460
+ allLayers: layerTypes.map((_, layerIdx) => layerIdx),
461
+ matchingLayers,
462
+ };
463
+ const qProjIndex = (result.decode || []).findIndex((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
464
+ if (qProjIndex === -1) {
465
+ return null;
466
+ }
467
+ const qProjStep = result.decode[qProjIndex];
468
+ const qProjKernelKey = qProjStep[1];
469
+ const qProjKernel = result.kernels[qProjKernelKey];
470
+ if (!qProjKernel) {
471
+ return null;
472
+ }
473
+
474
+ const derivedEntry = deriveLinearDecodeF16KernelEntry(qProjKernel);
475
+ if (!derivedEntry) {
476
+ return null;
477
+ }
478
+
479
+ const derivedKey = deriveKernelKey(result.kernels, qProjKernelKey, '_linear_f16out');
480
+ result.kernels[derivedKey] = derivedEntry;
481
+ const groupedEntries = buildGroupedLayerEntries(qProjStep, targetLayers, derivedKey);
482
+ if (groupedEntries.length === 0) {
483
+ return null;
484
+ }
485
+ result.decode = [
486
+ ...result.decode.slice(0, qProjIndex),
487
+ ...groupedEntries,
488
+ ...result.decode.slice(qProjIndex + 1),
489
+ ];
490
+
491
+ return result;
492
+ }
493
+
494
+ // =============================================================================
495
+ // Transform: remapQ4KDecodeToGemv
496
+ // =============================================================================
497
+
498
+ /*
499
+ * Replace fused Q4K decode projection kernels with GEMV subgroup variants.
500
+ *
501
+ * When Q4K weights have f16 materializations (mixed/dense loader mode), the
502
+ * GEMV subgroup kernel on pre-dequantized f16 weights is significantly faster
503
+ * than the fused Q4K kernel for M=1 decode (empirically 2.3x on Apple M-series).
504
+ *
505
+ * After this transform no decode kernels reference fused_matmul_q4*, which
506
+ * signals the loader to use dense materialization (f16 only — no Q4K buffer
507
+ * retained in GPU memory, reducing peak VRAM).
508
+ *
509
+ * Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
510
+ * attention, residual, activation) are left untouched.
511
+ *
512
+ */
513
+ export function remapQ4KDecodeToGemv(graph, ctx) {
514
+ if (ctx.activationDtype === 'f16') {
515
+ return null;
516
+ }
517
+
518
+ const decodeSteps = graph.decode || [];
519
+ const fusedDecodeKeys = new Set();
520
+ for (const step of decodeSteps) {
521
+ if (!Array.isArray(step)) continue;
522
+ const kernelKey = step[1];
523
+ const entry = graph.kernels[kernelKey];
524
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
525
+ fusedDecodeKeys.add(kernelKey);
526
+ }
527
+ }
528
+ if (fusedDecodeKeys.size === 0) {
529
+ return null;
530
+ }
531
+
532
+ const result = cloneGraph(graph);
533
+ const keyMap = new Map();
534
+
535
+ for (const key of fusedDecodeKeys) {
536
+ const newKey = deriveKernelKey(result.kernels, key, '_gemv');
537
+ result.kernels[newKey] = deriveKernelEntry(
538
+ result.kernels[key],
539
+ 'matmul_gemv_subgroup.wgsl',
540
+ 'main_multicol',
541
+ null
542
+ );
543
+ keyMap.set(key, newKey);
544
+ }
545
+
546
+ result.decode = remapStepKeys(result.decode, keyMap);
547
+ return result;
548
+ }
549
+
550
+ // =============================================================================
551
+ // Transform: remapQ4KDecodeAttentionToGemv (diagnostic)
552
+ // =============================================================================
553
+
554
+ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
555
+
556
+ /*
557
+ * Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
558
+ * subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
559
+ *
560
+ * Diagnostic transform for isolating whether the GEMV correctness regression
561
+ * originates in the attention or FFN projection path. Because FFN ops keep
562
+ * their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
563
+ * loader remains in mixed-materialization mode.
564
+ *
565
+ */
566
+ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
567
+ if (ctx.activationDtype === 'f16') {
568
+ return null;
569
+ }
570
+
571
+ const decodeSteps = graph.decode || [];
572
+ const attnFusedKeys = new Set();
573
+ for (const step of decodeSteps) {
574
+ if (!Array.isArray(step)) continue;
575
+ const op = step[0];
576
+ if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
577
+ const kernelKey = step[1];
578
+ const entry = graph.kernels[kernelKey];
579
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
580
+ attnFusedKeys.add(kernelKey);
581
+ }
582
+ }
583
+ if (attnFusedKeys.size === 0) {
584
+ return null;
585
+ }
586
+
587
+ const result = cloneGraph(graph);
588
+ const keyMap = new Map();
589
+
590
+ for (const key of attnFusedKeys) {
591
+ const newKey = deriveKernelKey(result.kernels, key, '_attn_gemv');
592
+ result.kernels[newKey] = deriveKernelEntry(
593
+ result.kernels[key],
594
+ 'matmul_gemv_subgroup.wgsl',
595
+ 'main_multicol',
596
+ null
597
+ );
598
+ keyMap.set(key, newKey);
599
+ }
600
+
601
+ // Only remap attention projection steps, leave FFN steps unchanged.
602
+ result.decode = result.decode.map((step) => {
603
+ if (!Array.isArray(step)) return step;
604
+ if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
605
+ const replacement = keyMap.get(step[1]);
606
+ if (replacement !== undefined) {
607
+ const newStep = [...step];
608
+ newStep[1] = replacement;
609
+ return newStep;
610
+ }
611
+ return step;
612
+ });
613
+
614
+ return result;
615
+ }
616
+
617
+ // =============================================================================
618
+ // Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
619
+ // =============================================================================
620
+
621
+ /*
622
+ * Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
623
+ * optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
624
+ * cooperative loading with fast nibble extraction for maximum M=1 throughput
625
+ * while preserving full Q4K dequant precision (no f16 weight materialization).
626
+ *
627
+ * This is the production fix for the f16-precision-loss regression observed
628
+ * when attention projections use the f16-weight GEMV path: softmax amplifies
629
+ * the f16 round-trip error in Q/K/V projections, causing garbage output.
630
+ * By keeping inline Q4K dequant (f32 arithmetic) the attention path stays
631
+ * numerically correct. FFN projections are unaffected and can safely use
632
+ * the f16-weight GEMV path via remapQ4KDecodeFFNToGemv.
633
+ *
634
+ * Because the derived kernel still references fused_matmul_q4.wgsl,
635
+ * isKernelPathFusedQ4K stays true and the weight loader remains in
636
+ * mixed-materialization mode (Q4K retained for attention, f16 for FFN).
637
+ *
638
+ */
639
+ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
640
+ if (ctx.activationDtype === 'f16') {
641
+ return null;
642
+ }
643
+
644
+ const decodeSteps = graph.decode || [];
645
+ const attnFusedKeys = new Set();
646
+ for (const step of decodeSteps) {
647
+ if (!Array.isArray(step)) continue;
648
+ const op = step[0];
649
+ if (!ATTENTION_PROJECTION_OPS.has(op)) continue;
650
+ const kernelKey = step[1];
651
+ const entry = graph.kernels[kernelKey];
652
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
653
+ attnFusedKeys.add(kernelKey);
654
+ }
655
+ }
656
+ if (attnFusedKeys.size === 0) {
657
+ return null;
658
+ }
659
+
660
+ const result = cloneGraph(graph);
661
+ const keyMap = new Map();
662
+
663
+ for (const key of attnFusedKeys) {
664
+ const newKey = deriveKernelKey(result.kernels, key, '_gemv');
665
+ result.kernels[newKey] = deriveKernelEntry(
666
+ result.kernels[key],
667
+ 'fused_matmul_q4.wgsl',
668
+ 'main_gemv',
669
+ null
670
+ );
671
+ keyMap.set(key, newKey);
672
+ }
673
+
674
+ // Only remap attention projection steps, leave FFN steps unchanged.
675
+ result.decode = result.decode.map((step) => {
676
+ if (!Array.isArray(step)) return step;
677
+ if (!ATTENTION_PROJECTION_OPS.has(step[0])) return step;
678
+ const replacement = keyMap.get(step[1]);
679
+ if (replacement !== undefined) {
680
+ const newStep = [...step];
681
+ newStep[1] = replacement;
682
+ return newStep;
683
+ }
684
+ return step;
685
+ });
686
+
687
+ return result;
688
+ }
689
+
690
+ // =============================================================================
691
+ // Transform: remapQ4KDecodeFFNToGemv (diagnostic)
692
+ // =============================================================================
693
+
694
+ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
695
+
696
+ /*
697
+ * Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
698
+ * variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
699
+ *
700
+ * Diagnostic complement to `remapQ4KDecodeAttentionToGemv`. Together these
701
+ * two transforms isolate whether the GEMV decode regression originates in
702
+ * the attention or FFN projection path. Because attention ops keep their
703
+ * fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
704
+ * remains in mixed-materialization mode.
705
+ *
706
+ */
707
+ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
708
+ if (ctx.activationDtype === 'f16') {
709
+ return null;
710
+ }
711
+
712
+ const decodeSteps = graph.decode || [];
713
+ const ffnFusedKeys = new Set();
714
+ for (const step of decodeSteps) {
715
+ if (!Array.isArray(step)) continue;
716
+ const op = step[0];
717
+ if (!FFN_PROJECTION_OPS.has(op)) continue;
718
+ const kernelKey = step[1];
719
+ const entry = graph.kernels[kernelKey];
720
+ if (entry && entry.kernel.startsWith('fused_matmul_q4')) {
721
+ ffnFusedKeys.add(kernelKey);
722
+ }
723
+ }
724
+ if (ffnFusedKeys.size === 0) {
725
+ return null;
726
+ }
727
+
728
+ const result = cloneGraph(graph);
729
+ const keyMap = new Map();
730
+
731
+ for (const key of ffnFusedKeys) {
732
+ const newKey = deriveKernelKey(result.kernels, key, '_ffn_gemv');
733
+ result.kernels[newKey] = deriveKernelEntry(
734
+ result.kernels[key],
735
+ 'matmul_gemv_subgroup.wgsl',
736
+ 'main_multicol',
737
+ null
738
+ );
739
+ keyMap.set(key, newKey);
740
+ }
741
+
742
+ // Only remap FFN projection steps, leave attention steps unchanged.
743
+ result.decode = result.decode.map((step) => {
744
+ if (!Array.isArray(step)) return step;
745
+ if (!FFN_PROJECTION_OPS.has(step[0])) return step;
746
+ const replacement = keyMap.get(step[1]);
747
+ if (replacement !== undefined) {
748
+ const newStep = [...step];
749
+ newStep[1] = replacement;
750
+ return newStep;
751
+ }
752
+ return step;
753
+ });
754
+
755
+ return result;
756
+ }
757
+
758
+ // =============================================================================
759
+ // Transform: useQwenDecodeF16Matmuls
760
+ // =============================================================================
761
+
762
+ /*
763
+ * Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
764
+ * kernels while keeping the manifest-wide f32 activation contract intact.
765
+ *
766
+ * This transform is intentionally selective:
767
+ * - FFN gate/up decode matmuls switch to f16a so decode can bypass the slow
768
+ * fused-q4k FFN path when capability policy opts in.
769
+ * - LM head decode switches to the subgroup f16a GEMV path.
770
+ *
771
+ * FFN down remains on the f32-output contract so the layer residual path stays
772
+ * numerically aligned with the manifest-owned activation dtype.
773
+ *
774
+ */
775
+
776
+ // =============================================================================
777
+ // Transform: useQwenF16PrimaryMatmuls
778
+ // =============================================================================
779
+
780
+ /*
781
+ * Select the Qwen 3.5 small-model execution graph's selective f16 probe lane
782
+ * when the runtime explicitly requests f16 activations.
783
+ *
784
+ * This transform narrows the decode projection and LM-head path while keeping
785
+ * `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
786
+ * already authored in the manifest on WideTile/head256 kernels and remains
787
+ * manifest-owned. The residual stream feeds directly into the next RMSNorm, and
788
+ * the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
789
+ * `o_proj` writes f16 there (observed first at the first full-attention block's
790
+ * post-attention RMSNorm).
791
+ *
792
+ */
793
+
794
+ // =============================================================================
795
+ // Transform: useQwen36F16Activations
796
+ // =============================================================================
797
+
798
+ /*
799
+ * Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
800
+ *
801
+ */
802
+
803
+ // =============================================================================
804
+ // Transform: useGemma4Int4PleSelectiveF16Decode
805
+ // =============================================================================
806
+
807
+ /*
808
+ * Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
809
+ * explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
810
+ * attention because the repository does not currently have pure-f16 head256 or
811
+ * head512 prefill kernels.
812
+ *
813
+ */
814
+
815
+ // =============================================================================
816
+ // Transform: useGemma4TextF16Activations
817
+ // =============================================================================
818
+
819
+ export function useGemma4TextF16Activations(graph, ctx) {
820
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
821
+ }
822
+
823
+ export function useGemma412BTextF16Activations(graph, ctx) {
824
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
825
+ }
826
+
827
+ export function useGemma431BTextF16Activations(graph, ctx) {
828
+ return useGemma4TextF16Activations(graph, ctx);
829
+ }
830
+
831
+ // =============================================================================
832
+ // Transform: useGemma4Int4PleAf16Activations
833
+ // =============================================================================
834
+
835
+ /*
836
+ * Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
837
+ * weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
838
+ * useGemma4TextF16Activations: same Q4 weight capsule, kernels narrowed to f16
839
+ * activations, prefill projections promoted from widetile to widetile_f16a,
840
+ * decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
841
+ * counterparts. Apple Metal stays disabled at the capability layer because the
842
+ * fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
843
+ *
844
+ */
845
+
846
+ // =============================================================================
847
+ // Composition
848
+ // =============================================================================
849
+
850
+ /*
851
+ * Compose multiple transforms into a single transform function.
852
+ *
853
+ * Each transform is applied sequentially. If a transform returns null
854
+ * (not applicable), the graph passes through unchanged.
855
+ *
856
+ */
857
+ export function composeTransforms(...transforms) {
858
+ return (graph, ctx) => {
859
+ let current = graph;
860
+ for (const transform of transforms) {
861
+ const result = transform(current, ctx);
862
+ if (result !== null && result !== undefined) {
863
+ current = result;
864
+ }
865
+ }
866
+ return current;
867
+ };
868
+ }
869
+
870
+ // Session-only capability transform marker. The execution-v1 compiler applies
871
+ // the matching runtime session patch; the graph itself is intentionally stable.
872
+ export function disableRetainQ4KMaterialization() {
873
+ return null;
874
+ }
875
+
876
+ /*
877
+ * Fail-closed sentinel transform. A capability rule installs this when the
878
+ * matched (modelId, runtime profile) combination is contradictory — for
879
+ * example, an af32 manifest variant paired with a runtime profile that
880
+ * demands f16 activations. The matcher reaches this transform only when the
881
+ * earlier manifest-binding gate has been bypassed; throwing here keeps the
882
+ * lane-confusion door shut at the capability layer too.
883
+ *
884
+ */
885
+ export function failClosedLaneMismatch(_graph, ctx) {
886
+ const modelId = ctx?.modelId ?? 'unknown';
887
+ const activationDtype = ctx?.activationDtype ?? 'unknown';
888
+ throw new Error(
889
+ `Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
890
+ 'The manifest variant tag is the lane identity — load the manifest variant ' +
891
+ 'whose compute lane matches the runtime profile.'
892
+ );
893
+ }
894
+
895
+ // =============================================================================
896
+ // Registry
897
+ // =============================================================================
898
+
899
+ export const TRANSFORMS = Object.freeze({
900
+ narrowToF16Activations,
901
+ removeSubgroups,
902
+ widenToF32Activations,
903
+ widenToF32CorrectnessFallback,
904
+ widenAttentionKvToF32,
905
+ swapPrefillAttention,
906
+ useHead256SmallPrefillAttention,
907
+ useHead256PrefillAttention,
908
+ widenProjectionWeightsToF32,
909
+ remapDenseQ4KPrefillToQ4Native,
910
+ remapQ4KPrefillToDense,
911
+ useLinearDecodeProjectionF16,
912
+ remapQ4KDecodeToGemv,
913
+ remapQ4KDecodeAttentionToGemv,
914
+ remapQ4KDecodeAttentionToFusedQ4KGemv,
915
+ remapQ4KDecodeFFNToGemv,
916
+ disableRetainQ4KMaterialization,
917
+ useQwenF16PrimaryMatmuls,
918
+ useQwen36F16Activations,
919
+ useQwenDecodeF16Matmuls,
920
+ useGemma4Int4PleSelectiveF16Decode,
921
+ useGemma4TextF16Activations,
922
+ useGemma412BTextF16Activations,
923
+ useGemma431BTextF16Activations,
924
+ useGemma4Int4PleAf16Activations,
925
+ failClosedLaneMismatch,
926
+ composeTransforms,
927
+ });