doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -13,16 +13,14 @@
13
13
  // Helpers
14
14
  // =============================================================================
15
15
 
16
- /**
16
+ /*
17
17
  * Deep-clone an execution graph.
18
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
19
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph}
20
18
  */
21
19
  function cloneGraph(graph) {
22
20
  return structuredClone(graph);
23
21
  }
24
22
 
25
- /**
23
+ /*
26
24
  * Shader files that require subgroups even though "subgroup" is not in the filename.
27
25
  * Online attention kernels use subgroup reductions internally.
28
26
  */
@@ -51,10 +49,8 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
51
49
  ['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
52
50
  ]);
53
51
 
54
- /**
52
+ /*
55
53
  * Check whether a kernel entry requires subgroup support.
56
- * @param {{ kernel: string }} kernelEntry
57
- * @returns {boolean}
58
54
  */
59
55
  function isSubgroupKernel(kernelEntry) {
60
56
  if (typeof kernelEntry.kernel !== 'string') return false;
@@ -66,11 +62,8 @@ function requiresNoSubgroupFallback(kernelEntry) {
66
62
  return isSubgroupKernel(kernelEntry) || kernelEntry.kernel.startsWith('fused_matmul_q4');
67
63
  }
68
64
 
69
- /**
65
+ /*
70
66
  * Find all kernel keys in the graph whose `kernel` file matches the given filename.
71
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
72
- * @param {string} filename
73
- * @returns {string[]}
74
67
  */
75
68
  function findKernelKeysByFile(graph, filename) {
76
69
  const keys = [];
@@ -82,23 +75,15 @@ function findKernelKeysByFile(graph, filename) {
82
75
  return keys;
83
76
  }
84
77
 
85
- /**
78
+ /*
86
79
  * Check whether any kernel in the graph uses the given shader file.
87
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
88
- * @param {string} filename
89
- * @returns {boolean}
90
80
  */
91
81
  function hasKernelFile(graph, filename) {
92
82
  return findKernelKeysByFile(graph, filename).length > 0;
93
83
  }
94
84
 
95
- /**
85
+ /*
96
86
  * Create a new kernel entry with the digest cleared (shader changed).
97
- * @param {object} base - original kernel entry
98
- * @param {string} newFile - new shader filename
99
- * @param {string} newEntry - new entry point name
100
- * @param {object|null} [constants] - override constants (null to remove)
101
- * @returns {object}
102
87
  */
103
88
  function deriveKernelEntry(base, newFile, newEntry, constants) {
104
89
  const derived = { ...base, kernel: newFile, entry: newEntry, digest: null };
@@ -145,12 +130,8 @@ function deriveKernelPrecision(base, newFile) {
145
130
  return precision;
146
131
  }
147
132
 
148
- /**
133
+ /*
149
134
  * Derive a non-colliding kernel key name.
150
- * @param {object} kernels - existing kernels dict
151
- * @param {string} baseKey - original key
152
- * @param {string} suffix - suffix to append
153
- * @returns {string}
154
135
  */
155
136
  function deriveKernelKey(kernels, baseKey, suffix) {
156
137
  const candidate = `${baseKey}${suffix}`;
@@ -164,11 +145,8 @@ function deriveKernelKey(kernels, baseKey, suffix) {
164
145
  return `${candidate}_${counter}`;
165
146
  }
166
147
 
167
- /**
148
+ /*
168
149
  * Replace kernel key references in step tuples.
169
- * @param {Array<Array>} steps
170
- * @param {Map<string, string>} keyMap - oldKey → newKey
171
- * @returns {Array<Array>}
172
150
  */
173
151
  function remapStepKeys(steps, keyMap) {
174
152
  return steps.map((step) => {
@@ -183,12 +161,8 @@ function remapStepKeys(steps, keyMap) {
183
161
  });
184
162
  }
185
163
 
186
- /**
164
+ /*
187
165
  * Check whether a step tuple's kernel key resolves to the given shader file.
188
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
189
- * @param {Array} step
190
- * @param {string} filename
191
- * @returns {boolean}
192
166
  */
193
167
  function stepUsesFile(graph, step, filename) {
194
168
  const kernelKey = step[1];
@@ -196,14 +170,9 @@ function stepUsesFile(graph, step, filename) {
196
170
  return entry != null && entry.kernel === filename;
197
171
  }
198
172
 
199
- /**
173
+ /*
200
174
  * Find the first kernel key used by matching ops in a phase whose shader file
201
175
  * satisfies the provided predicate.
202
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
203
- * @param {Array<Array>} steps
204
- * @param {ReadonlySet<string>} ops
205
- * @param {(entry: { kernel: string, entry: string }) => boolean} predicate
206
- * @returns {string | null}
207
176
  */
208
177
  function findPhaseKernelKey(graph, steps, ops, predicate) {
209
178
  for (const step of steps || []) {
@@ -218,12 +187,8 @@ function findPhaseKernelKey(graph, steps, ops, predicate) {
218
187
  return null;
219
188
  }
220
189
 
221
- /**
190
+ /*
222
191
  * Find an existing kernel key by shader file and entry point.
223
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
224
- * @param {string} filename
225
- * @param {string} entryPoint
226
- * @returns {string | null}
227
192
  */
228
193
  function findKernelKeyByFileAndEntry(graph, filename, entryPoint) {
229
194
  for (const [key, entry] of Object.entries(graph.kernels)) {
@@ -544,10 +509,25 @@ function deriveQ4DecodeF32ActivationKernelEntry(base) {
544
509
  return null;
545
510
  }
546
511
 
547
- function deriveQ4PrefillF32ActivationKernelEntry(base) {
512
+ function deriveQ4PrefillF32ActivationKernelEntry(base, options = {}) {
548
513
  if (typeof base?.kernel !== 'string') {
549
514
  return null;
550
515
  }
516
+ const replacement = options.fullF32 === true
517
+ ? {
518
+ kernel: 'fused_matmul_q4_batched_multicol_shared.wgsl',
519
+ entry: 'main',
520
+ }
521
+ : {
522
+ kernel: 'fused_matmul_q4_widetile.wgsl',
523
+ entry: 'main',
524
+ };
525
+ if (base.kernel === 'fused_matmul_q4_widetile.wgsl' && options.fullF32 === true) {
526
+ return deriveKernelEntryWithPrecision(
527
+ deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
528
+ { inputDtype: 'f32', outputDtype: 'f32' }
529
+ );
530
+ }
551
531
  if (
552
532
  base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
553
533
  && base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
@@ -557,7 +537,7 @@ function deriveQ4PrefillF32ActivationKernelEntry(base) {
557
537
  }
558
538
  if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
559
539
  return deriveKernelEntryWithPrecision(
560
- deriveKernelEntry(base, 'fused_matmul_q4_widetile.wgsl', 'main', null),
540
+ deriveKernelEntry(base, replacement.kernel, replacement.entry, null),
561
541
  { inputDtype: 'f32', outputDtype: 'f32' }
562
542
  );
563
543
  }
@@ -589,15 +569,12 @@ function replacePhaseStepEntries(steps, op, replacementEntries) {
589
569
  // Transform: removeSubgroups
590
570
  // =============================================================================
591
571
 
592
- /**
572
+ /*
593
573
  * Remove subgroup shader dependencies from decode and postLayer steps.
594
574
  * Prefill steps are left untouched (they already use tiled matmul).
595
575
  *
596
576
  * Returns null if the graph has no subgroup kernels.
597
577
  *
598
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
599
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
600
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
601
578
  */
602
579
  export function removeSubgroups(graph, ctx) {
603
580
  const hasAnyFallbackKernel = Object.values(graph.kernels).some(requiresNoSubgroupFallback);
@@ -684,11 +661,10 @@ export function removeSubgroups(graph, ctx) {
684
661
  // Transform: widenToF32Activations
685
662
  // =============================================================================
686
663
 
687
- /**
664
+ /*
688
665
  * Activation-only widening: f16-activation shaders → f32-activation variants
689
666
  * that still use f16 for weights and KV cache. Requires shader-f16 for weight
690
667
  * and KV buffer reads.
691
- * @type {ReadonlyMap<string, string>}
692
668
  */
693
669
  const F16_TO_F32_ACTIVATION_MAP = new Map([
694
670
  ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
@@ -710,14 +686,13 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
710
686
  ['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
711
687
  ]);
712
688
 
713
- /**
689
+ /*
714
690
  * Activation-only narrowing: f32-activation shaders that still consume f16
715
691
  * weights/KV are rewritten onto the matching f16-activation lane.
716
692
  *
717
693
  * This is the inverse of `F16_TO_F32_ACTIVATION_MAP` and is used when a
718
694
  * runtime session explicitly requests f16 activations for an execution-v1
719
695
  * graph that was authored with conservative f32 activation defaults.
720
- * @type {ReadonlyMap<string, string>}
721
696
  */
722
697
  const F32_TO_F16_ACTIVATION_MAP = new Map(
723
698
  Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
@@ -738,11 +713,10 @@ function hasExplicitF32ActivationContract(entry) {
738
713
  || precision.outputDtype === 'f32';
739
714
  }
740
715
 
741
- /**
716
+ /*
742
717
  * Correctness fallback: preserve f16 weights where possible, but widen both
743
718
  * activations and KV-cache interactions onto the stable f32 execution lane.
744
719
  * Used for alternate-plan recovery after finiteness failure.
745
- * @type {ReadonlyMap<string, string>}
746
720
  */
747
721
  const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
748
722
  ['rmsnorm_f16.wgsl', 'rmsnorm.wgsl'],
@@ -767,11 +741,10 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
767
741
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
768
742
  ]);
769
743
 
770
- /**
744
+ /*
771
745
  * Full f32 widening: every shader that uses `enable f16;` is replaced with a
772
746
  * pure-f32 equivalent. Used when the GPU cannot compile any f16 WGSL at all.
773
747
  * Covers f16-activation, f16-weight (f16w), and f16-KV (f16kv) kernels.
774
- * @type {ReadonlyMap<string, string>}
775
748
  */
776
749
  const FULL_F32_SHADER_MAP = new Map([
777
750
  // f16-activation utility kernels → f32
@@ -805,7 +778,7 @@ const FULL_F32_SHADER_MAP = new Map([
805
778
  ['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
806
779
  ]);
807
780
 
808
- /**
781
+ /*
809
782
  * Widen all f16-activation shaders to f32-activation equivalents.
810
783
  *
811
784
  * Returns null if the graph contains fused_ffn_f16.wgsl (no direct f32
@@ -814,9 +787,6 @@ const FULL_F32_SHADER_MAP = new Map([
814
787
  * NOTE: The caller is responsible for also updating session.activationDtype
815
788
  * to reflect the widened dtype.
816
789
  *
817
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
818
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
819
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
820
790
  */
821
791
  export function widenToF32Activations(graph, ctx) {
822
792
  // Bail out if fused f16 FFN is present — no direct f32 equivalent
@@ -830,9 +800,12 @@ export function widenToF32Activations(graph, ctx) {
830
800
  const shaderMap = ctx.capabilities?.hasF16 === false
831
801
  ? FULL_F32_SHADER_MAP
832
802
  : F16_TO_F32_ACTIVATION_MAP;
803
+ const fullF32 = ctx.capabilities?.hasF16 === false;
833
804
 
834
805
  const hasTargetShader = Object.values(graph.kernels).some(
835
806
  (entry) => shaderMap.has(entry.kernel)
807
+ || deriveQ4DecodeF32ActivationKernelEntry(entry)
808
+ || deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 })
836
809
  );
837
810
  if (!hasTargetShader) {
838
811
  return null;
@@ -842,7 +815,7 @@ export function widenToF32Activations(graph, ctx) {
842
815
 
843
816
  for (const [key, entry] of Object.entries(result.kernels)) {
844
817
  const q4FallbackEntry = deriveQ4DecodeF32ActivationKernelEntry(entry)
845
- ?? deriveQ4PrefillF32ActivationKernelEntry(entry);
818
+ ?? deriveQ4PrefillF32ActivationKernelEntry(entry, { fullF32 });
846
819
  if (q4FallbackEntry) {
847
820
  result.kernels[key] = q4FallbackEntry;
848
821
  continue;
@@ -856,13 +829,10 @@ export function widenToF32Activations(graph, ctx) {
856
829
  return result;
857
830
  }
858
831
 
859
- /**
832
+ /*
860
833
  * Widen an f16 execution graph onto the stable f32 correctness lane used for
861
834
  * alternate-plan recovery after finiteness failure.
862
835
  *
863
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
864
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
865
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
866
836
  */
867
837
  export function widenToF32CorrectnessFallback(graph, ctx) {
868
838
  if (hasKernelFile(graph, 'fused_ffn_f16.wgsl')) {
@@ -892,16 +862,13 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
892
862
  return result;
893
863
  }
894
864
 
895
- /**
865
+ /*
896
866
  * Narrow f32-activation shaders back onto their f16-activation equivalents.
897
867
  *
898
868
  * Returns null if the graph has no supported f32-activation kernels to swap or
899
869
  * if the runtime did not explicitly request f16 activations on an f16-capable
900
870
  * GPU.
901
871
  *
902
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
903
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
904
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
905
872
  */
906
873
  export function narrowToF16Activations(graph, ctx) {
907
874
  if (ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
@@ -932,7 +899,6 @@ export function narrowToF16Activations(graph, ctx) {
932
899
  // Transform: swapPrefillAttention
933
900
  // =============================================================================
934
901
 
935
- /** @type {ReadonlyMap<string, string>} */
936
902
  const PREFILL_ATTENTION_PAIRS = new Map([
937
903
  ['attention_streaming_f16kv.wgsl', 'attention_small_f16kv.wgsl'],
938
904
  ['attention_small_f16kv.wgsl', 'attention_streaming_f16kv.wgsl'],
@@ -960,7 +926,7 @@ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
960
926
  return false;
961
927
  }
962
928
 
963
- /**
929
+ /*
964
930
  * Swap prefill attention kernel between streaming and small variants.
965
931
  *
966
932
  * The `opts` parameter specifies the direction:
@@ -969,10 +935,6 @@ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
969
935
  * If `from`/`to` are not provided, uses the bidirectional pair map.
970
936
  * Returns null if no matching prefill attention kernel is found.
971
937
  *
972
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
973
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
974
- * @param {{ from?: string, to?: string }} [opts]
975
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
976
938
  */
977
939
  export function swapPrefillAttention(graph, ctx, opts) {
978
940
  const from = opts?.from;
@@ -1010,12 +972,9 @@ export function swapPrefillAttention(graph, ctx, opts) {
1010
972
  // Transform: useHead256PrefillAttention
1011
973
  // =============================================================================
1012
974
 
1013
- /**
975
+ /*
1014
976
  * Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
1015
977
  *
1016
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1017
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1018
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1019
978
  */
1020
979
  export function useHead256SmallPrefillAttention(graph, ctx) {
1021
980
  return swapPrefillAttention(graph, ctx, {
@@ -1024,12 +983,9 @@ export function useHead256SmallPrefillAttention(graph, ctx) {
1024
983
  });
1025
984
  }
1026
985
 
1027
- /**
986
+ /*
1028
987
  * Promote prefill attention onto the fixed 256-dim shared-block kernel.
1029
988
  *
1030
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1031
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1032
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1033
989
  */
1034
990
  export function useHead256PrefillAttention(graph, ctx) {
1035
991
  let current = graph;
@@ -1057,7 +1013,6 @@ export function useHead256PrefillAttention(graph, ctx) {
1057
1013
  // Transform: widenProjectionWeightsToF32
1058
1014
  // =============================================================================
1059
1015
 
1060
- /** @type {ReadonlySet<string>} */
1061
1016
  const PROJECTION_MATMUL_FILES = new Set([
1062
1017
  'matmul_gemv_subgroup.wgsl',
1063
1018
  'matmul_gemv_subgroup_f16a.wgsl',
@@ -1067,10 +1022,9 @@ const PROJECTION_MATMUL_FILES = new Set([
1067
1022
  'matmul_f16_tiled.wgsl',
1068
1023
  ]);
1069
1024
 
1070
- /**
1025
+ /*
1071
1026
  * Known layer projection ops. Only these are widened; lm_head and embed are
1072
1027
  * excluded.
1073
- * @type {ReadonlySet<string>}
1074
1028
  */
1075
1029
  const LAYER_PROJECTION_OPS = new Set([
1076
1030
  'q_proj', 'k_proj', 'v_proj', 'o_proj',
@@ -1090,7 +1044,7 @@ function resolveDensePrefillProjectionKernel(ctx) {
1090
1044
  : 'matmul_f16w_f32a.wgsl';
1091
1045
  }
1092
1046
 
1093
- /**
1047
+ /*
1094
1048
  * Replace projection matmul kernels with f32 weight variants.
1095
1049
  *
1096
1050
  * Applies only to layer projection steps (q/k/v/o/gate/up/down), NOT lm_head
@@ -1098,9 +1052,6 @@ function resolveDensePrefillProjectionKernel(ctx) {
1098
1052
  *
1099
1053
  * Returns null if no applicable projection kernels are found.
1100
1054
  *
1101
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1102
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1103
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1104
1055
  */
1105
1056
  export function widenProjectionWeightsToF32(graph, ctx) {
1106
1057
  // Collect kernel keys used by layer projection steps across all phases
@@ -1152,7 +1103,7 @@ export function widenProjectionWeightsToF32(graph, ctx) {
1152
1103
  // Transform: remapDenseQ4KPrefillToQ4Native
1153
1104
  // =============================================================================
1154
1105
 
1155
- /**
1106
+ /*
1156
1107
  * Replace dense prefill projection kernels with Q4-native prefill variants.
1157
1108
  *
1158
1109
  * This applies only when the graph already exposes a compatible fused Q4 decode
@@ -1163,9 +1114,6 @@ export function widenProjectionWeightsToF32(graph, ctx) {
1163
1114
  * Returns null when the graph does not have the required dense-prefill + Q4
1164
1115
  * decode shape.
1165
1116
  *
1166
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1167
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1168
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1169
1117
  */
1170
1118
  export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1171
1119
  const densePrefillProjectionSteps = (graph.prefill || []).filter((step) => {
@@ -1234,7 +1182,7 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1234
1182
  // Transform: remapQ4KPrefillToDense
1235
1183
  // =============================================================================
1236
1184
 
1237
- /**
1185
+ /*
1238
1186
  * Replace fused Q4K prefill projection kernels with dense tiled variants.
1239
1187
  *
1240
1188
  * Decode remains unchanged so the runtime can keep using fused Q4K decode while
@@ -1242,9 +1190,6 @@ export function remapDenseQ4KPrefillToQ4Native(graph, ctx) {
1242
1190
  *
1243
1191
  * Returns null when the graph has no fused Q4K prefill projection kernels.
1244
1192
  *
1245
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1246
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1247
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1248
1193
  */
1249
1194
  export function remapQ4KPrefillToDense(graph, ctx) {
1250
1195
  const q4PrefillProjectionSteps = (graph.prefill || []).filter((step) => {
@@ -1298,7 +1243,7 @@ export function remapQ4KPrefillToDense(graph, ctx) {
1298
1243
  // Transform: useLinearDecodeProjectionF16
1299
1244
  // =============================================================================
1300
1245
 
1301
- /**
1246
+ /*
1302
1247
  * Remap the linear-attention q_proj decode step onto the f16-activation fused
1303
1248
  * Q4 kernel for linear-attention layers only. Full-attention layers keep the
1304
1249
  * manifest-wide f32 activation contract.
@@ -1310,9 +1255,6 @@ export function remapQ4KPrefillToDense(graph, ctx) {
1310
1255
  * repetitive output under greedy decode). q_proj f16 is safe because the
1311
1256
  * linear attention core absorbs the f16 input into its f32 internal state.
1312
1257
  *
1313
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1314
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1315
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1316
1258
  */
1317
1259
  export function useLinearDecodeProjectionF16(graph, ctx) {
1318
1260
  const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
@@ -1368,7 +1310,7 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
1368
1310
  // Transform: remapQ4KDecodeToGemv
1369
1311
  // =============================================================================
1370
1312
 
1371
- /**
1313
+ /*
1372
1314
  * Replace fused Q4K decode projection kernels with GEMV subgroup variants.
1373
1315
  *
1374
1316
  * When Q4K weights have f16 materializations (mixed/dense loader mode), the
@@ -1382,9 +1324,6 @@ export function useLinearDecodeProjectionF16(graph, ctx) {
1382
1324
  * Only layer projection ops are remapped. Non-matmul ops (rmsnorm, rope,
1383
1325
  * attention, residual, activation) are left untouched.
1384
1326
  *
1385
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1386
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1387
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1388
1327
  */
1389
1328
  export function remapQ4KDecodeToGemv(graph, ctx) {
1390
1329
  if (ctx.activationDtype === 'f16') {
@@ -1429,7 +1368,7 @@ export function remapQ4KDecodeToGemv(graph, ctx) {
1429
1368
 
1430
1369
  const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj']);
1431
1370
 
1432
- /**
1371
+ /*
1433
1372
  * Replace fused Q4K ATTENTION-ONLY decode projection kernels with GEMV
1434
1373
  * subgroup variants, leaving FFN projections (gate/up/down_proj) untouched.
1435
1374
  *
@@ -1438,9 +1377,6 @@ const ATTENTION_PROJECTION_OPS = new Set(['q_proj', 'k_proj', 'v_proj', 'o_proj'
1438
1377
  * their fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight
1439
1378
  * loader remains in mixed-materialization mode.
1440
1379
  *
1441
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1442
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1443
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1444
1380
  */
1445
1381
  export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1446
1382
  if (ctx.activationDtype === 'f16') {
@@ -1497,7 +1433,7 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1497
1433
  // Transform: remapQ4KDecodeAttentionToFusedQ4KGemv
1498
1434
  // =============================================================================
1499
1435
 
1500
- /**
1436
+ /*
1501
1437
  * Replace fused Q4K ATTENTION-ONLY decode projection kernels with the
1502
1438
  * optimised fused Q4K GEMV variant (main_gemv), which combines shared-A
1503
1439
  * cooperative loading with fast nibble extraction for maximum M=1 throughput
@@ -1514,9 +1450,6 @@ export function remapQ4KDecodeAttentionToGemv(graph, ctx) {
1514
1450
  * isKernelPathFusedQ4K stays true and the weight loader remains in
1515
1451
  * mixed-materialization mode (Q4K retained for attention, f16 for FFN).
1516
1452
  *
1517
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1518
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1519
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1520
1453
  */
1521
1454
  export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
1522
1455
  if (ctx.activationDtype === 'f16') {
@@ -1575,7 +1508,7 @@ export function remapQ4KDecodeAttentionToFusedQ4KGemv(graph, ctx) {
1575
1508
 
1576
1509
  const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
1577
1510
 
1578
- /**
1511
+ /*
1579
1512
  * Replace fused Q4K FFN-ONLY decode projection kernels with GEMV subgroup
1580
1513
  * variants, leaving attention projections (q/k/v/o_proj) as fused Q4K.
1581
1514
  *
@@ -1585,9 +1518,6 @@ const FFN_PROJECTION_OPS = new Set(['gate_proj', 'up_proj', 'down_proj']);
1585
1518
  * fused Q4K kernels, `isKernelPathFusedQ4K` stays true and the weight loader
1586
1519
  * remains in mixed-materialization mode.
1587
1520
  *
1588
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1589
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1590
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1591
1521
  */
1592
1522
  export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1593
1523
  if (ctx.activationDtype === 'f16') {
@@ -1644,7 +1574,7 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1644
1574
  // Transform: useQwenDecodeF16Matmuls
1645
1575
  // =============================================================================
1646
1576
 
1647
- /**
1577
+ /*
1648
1578
  * Narrow selected Qwen decode matmuls onto explicit f16-input/f16-output
1649
1579
  * kernels while keeping the manifest-wide f32 activation contract intact.
1650
1580
  *
@@ -1656,16 +1586,8 @@ export function remapQ4KDecodeFFNToGemv(graph, ctx) {
1656
1586
  * FFN down remains on the f32-output contract so the layer residual path stays
1657
1587
  * numerically aligned with the manifest-owned activation dtype.
1658
1588
  *
1659
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1660
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1661
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1662
1589
  */
1663
1590
  export function useQwenDecodeF16Matmuls(graph, ctx) {
1664
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1665
- if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16') {
1666
- return null;
1667
- }
1668
-
1669
1591
  const result = cloneGraph(graph);
1670
1592
  let changed = false;
1671
1593
 
@@ -1730,7 +1652,7 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
1730
1652
  // Transform: useQwenF16PrimaryMatmuls
1731
1653
  // =============================================================================
1732
1654
 
1733
- /**
1655
+ /*
1734
1656
  * Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
1735
1657
  * lane when the runtime explicitly requests f16 activations.
1736
1658
  *
@@ -1742,16 +1664,8 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
1742
1664
  * `o_proj` writes f16 there (observed first at the first full-attention block's
1743
1665
  * post-attention RMSNorm).
1744
1666
  *
1745
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1746
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1747
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1748
1667
  */
1749
1668
  export function useQwenF16PrimaryMatmuls(graph, ctx) {
1750
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1751
- if (modelId !== 'qwen-3-5-0-8b-q4k-ehaf16' || ctx.activationDtype !== 'f16' || ctx.capabilities?.hasF16 !== true) {
1752
- return null;
1753
- }
1754
-
1755
1669
  const layerTypes = Array.isArray(ctx.layerTypes) ? ctx.layerTypes : null;
1756
1670
  if (!layerTypes || layerTypes.length === 0) {
1757
1671
  return null;
@@ -1848,26 +1762,11 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
1848
1762
  // Transform: useQwen36F16Activations
1849
1763
  // =============================================================================
1850
1764
 
1851
- /**
1765
+ /*
1852
1766
  * Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
1853
1767
  *
1854
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1855
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1856
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1857
1768
  */
1858
1769
  export function useQwen36F16Activations(graph, ctx) {
1859
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1860
- if (
1861
- modelId !== 'qwen-3-6-27b-q4k-eaf16'
1862
- || ctx.activationDtype !== 'f16'
1863
- || ctx.mathDtype !== 'f16'
1864
- || ctx.accumDtype !== 'f16'
1865
- || ctx.capabilities?.hasF16 !== true
1866
- || ctx.capabilities?.hasSubgroups !== true
1867
- ) {
1868
- return null;
1869
- }
1870
-
1871
1770
  const narrowed = narrowToF16Activations(graph, ctx);
1872
1771
  const result = narrowed ?? cloneGraph(graph);
1873
1772
  let changed = narrowed != null;
@@ -1914,8 +1813,8 @@ export function useQwen36F16Activations(graph, ctx) {
1914
1813
  const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
1915
1814
  const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
1916
1815
  const prefillProjectionEntry =
1917
- deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
1918
- ?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
1816
+ deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
1817
+ ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
1919
1818
  ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
1920
1819
  if (prefillProjectionEntry) {
1921
1820
  const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
@@ -1933,7 +1832,13 @@ export function useQwen36F16Activations(graph, ctx) {
1933
1832
  const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
1934
1833
  const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
1935
1834
  const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
1936
- if (
1835
+ const lmHeadPrefillF16Entry =
1836
+ deriveQ4WideTilePrefillF16KernelEntry(lmHeadPrefillEntry)
1837
+ ?? deriveQ4PrefillF16AccumKernelEntry(lmHeadPrefillEntry)
1838
+ ?? deriveQ4PrefillF16KernelEntry(lmHeadPrefillEntry);
1839
+ if (lmHeadPrefillF16Entry) {
1840
+ replaceKernelEntry(lmHeadPrefillKey, lmHeadPrefillF16Entry);
1841
+ } else if (
1937
1842
  lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
1938
1843
  || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
1939
1844
  ) {
@@ -1953,27 +1858,14 @@ export function useQwen36F16Activations(graph, ctx) {
1953
1858
  // Transform: useGemma4Int4PleSelectiveF16Decode
1954
1859
  // =============================================================================
1955
1860
 
1956
- /**
1861
+ /*
1957
1862
  * Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
1958
1863
  * explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
1959
1864
  * attention because the repository does not currently have pure-f16 head256 or
1960
1865
  * head512 prefill kernels.
1961
1866
  *
1962
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1963
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1964
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1965
1867
  */
1966
1868
  export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
1967
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1968
- if (
1969
- modelId !== 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple'
1970
- || ctx.activationDtype !== 'f16'
1971
- || ctx.capabilities?.hasF16 !== true
1972
- || ctx.capabilities?.hasSubgroups !== true
1973
- ) {
1974
- return null;
1975
- }
1976
-
1977
1869
  const result = cloneGraph(graph);
1978
1870
  let changed = false;
1979
1871
 
@@ -2044,35 +1936,95 @@ export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
2044
1936
  }
2045
1937
 
2046
1938
  // =============================================================================
2047
- // Transform: useGemma431BTextF16Activations
1939
+ // Transform: useGemma4TextF16Activations
2048
1940
  // =============================================================================
2049
1941
 
2050
- /**
2051
- * Promote the Gemma 4 31B text Q4K graph onto an experimental all-f16 request
2052
- * lane. This keeps the same Q4K weight pack but remaps execution-v1 kernels so
2053
- * Q4 projections, attention, residual/norm/activation utilities, lm_head, and
2054
- * sampling consume and produce f16 activations where a matching kernel exists.
2055
- *
2056
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
2057
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2058
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
2059
- */
2060
- export function useGemma431BTextF16Activations(graph, ctx) {
2061
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
2062
- if (
2063
- ![
2064
- 'gemma-4-31b-it-text-q4k-ehf16-af32',
2065
- 'gemma-4-31b-it-text-q4k-ehf16-af16',
2066
- ].includes(modelId)
2067
- || ctx.activationDtype !== 'f16'
2068
- || ctx.mathDtype !== 'f16'
2069
- || ctx.accumDtype !== 'f16'
2070
- || ctx.capabilities?.hasF16 !== true
2071
- || ctx.capabilities?.hasSubgroups !== true
2072
- ) {
2073
- return null;
1942
+ const GEMMA4_12B_PREFILL_F32_PROJECTION_OPS = new Set([
1943
+ 'q_proj',
1944
+ 'k_proj',
1945
+ 'v_proj',
1946
+ 'gate_proj',
1947
+ 'up_proj',
1948
+ ]);
1949
+
1950
+ function remapGemma412BPrefillProjectionEntries(entries, f16Key, f32Key) {
1951
+ if (!Array.isArray(entries) || !f16Key || !f32Key) {
1952
+ return { entries, changed: false };
2074
1953
  }
2075
1954
 
1955
+ let changed = false;
1956
+ const remapStep = (step) => {
1957
+ if (!Array.isArray(step) || !LAYER_PROJECTION_OPS.has(step[0])) {
1958
+ return step;
1959
+ }
1960
+ const targetKey = GEMMA4_12B_PREFILL_F32_PROJECTION_OPS.has(step[0])
1961
+ ? f32Key
1962
+ : f16Key;
1963
+ if (step[1] === targetKey) {
1964
+ return step;
1965
+ }
1966
+ const replacement = [...step];
1967
+ replacement[1] = targetKey;
1968
+ changed = true;
1969
+ return replacement;
1970
+ };
1971
+
1972
+ const nextEntries = entries.map((entry) => {
1973
+ if (Array.isArray(entry)) {
1974
+ return remapStep(entry);
1975
+ }
1976
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
1977
+ return entry;
1978
+ }
1979
+ return {
1980
+ ...entry,
1981
+ steps: entry.steps.map((step) => remapStep(step)),
1982
+ };
1983
+ });
1984
+
1985
+ return { entries: nextEntries, changed };
1986
+ }
1987
+
1988
+ function remapGemma412BStableBoundaryEntries(result, sourceGraph) {
1989
+ let changed = false;
1990
+
1991
+ const replaceStepWithSourceEntry = (phaseName, op, precision) => {
1992
+ const phase = result[phaseName];
1993
+ const sourcePhase = sourceGraph[phaseName];
1994
+ const step = findPhaseStep(phase, op);
1995
+ const sourceStep = findPhaseStep(sourcePhase, op);
1996
+ const sourceKey = sourceStep?.[1] ?? step?.[1] ?? null;
1997
+ const sourceEntry = sourceKey ? sourceGraph.kernels[sourceKey] : null;
1998
+ if (!step || !sourceKey || !sourceEntry) {
1999
+ return;
2000
+ }
2001
+ const stableKey = deriveKernelKey(result.kernels, sourceKey, '_gemma4_12b_stable');
2002
+ result.kernels[stableKey] = precision
2003
+ ? deriveKernelEntryWithPrecision(sourceEntry, precision)
2004
+ : { ...sourceEntry };
2005
+ const phaseResult = replacePhaseStepKernelKey(phase, op, stableKey);
2006
+ if (phaseResult.changed) {
2007
+ result[phaseName] = phaseResult.steps;
2008
+ changed = true;
2009
+ }
2010
+ };
2011
+
2012
+ for (const op of ['q_proj', 'k_proj', 'v_proj', 'rope_q', 'rope_k']) {
2013
+ replaceStepWithSourceEntry('decode', op, { inputDtype: 'f32', outputDtype: 'f32' });
2014
+ }
2015
+ replaceStepWithSourceEntry('decode', 'attention', {
2016
+ activationDtype: 'f32',
2017
+ kvDtype: 'f16',
2018
+ outputDtype: 'f32',
2019
+ });
2020
+ for (const op of ['final_norm', 'lm_head', 'lm_head_prefill', 'sample']) {
2021
+ replaceStepWithSourceEntry('postLayer', op, null);
2022
+ }
2023
+
2024
+ return changed;
2025
+ }
2026
+
2027
+ function useGemma4TextF16ActivationsForLane(graph, ctx, options) {
2076
2028
  const narrowed = narrowToF16Activations(graph, ctx);
2077
2029
  const result = narrowed ?? cloneGraph(graph);
2078
2030
  let changed = narrowed != null;
@@ -2084,6 +2036,31 @@ export function useGemma431BTextF16Activations(graph, ctx) {
2084
2036
  result.kernels[key] = entry;
2085
2037
  changed = true;
2086
2038
  };
2039
+ const stableTextBoundary = options?.stableTextBoundary === true;
2040
+
2041
+ const derivePrefillAttentionEntry = (entry) => {
2042
+ if (
2043
+ stableTextBoundary
2044
+ && typeof entry?.kernel === 'string'
2045
+ && entry.kernel === 'attention_small_f16.wgsl'
2046
+ ) {
2047
+ return deriveKernelEntryWithPrecision(
2048
+ deriveKernelEntry(entry, 'attention_head256_f16kv.wgsl', 'main'),
2049
+ { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2050
+ );
2051
+ }
2052
+ if (
2053
+ stableTextBoundary
2054
+ && typeof entry?.kernel === 'string'
2055
+ && entry.kernel.endsWith('_f16kv.wgsl')
2056
+ ) {
2057
+ return deriveKernelEntryWithPrecision(
2058
+ entry,
2059
+ { activationDtype: 'f32', kvDtype: 'f16', outputDtype: 'f32' }
2060
+ );
2061
+ }
2062
+ return deriveF16AttentionKernelEntry(entry);
2063
+ };
2087
2064
 
2088
2065
  const embedStep = findPhaseStep(result.preLayer, 'embed');
2089
2066
  const embedKey = embedStep?.[1] ?? null;
@@ -2107,12 +2084,35 @@ export function useGemma431BTextF16Activations(graph, ctx) {
2107
2084
 
2108
2085
  const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2109
2086
  const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2110
- replaceKernelEntry(
2111
- prefillProjectionKey,
2112
- deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
2113
- ?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2114
- ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2115
- );
2087
+ const sourcePrefillProjectionEntry = result.kernels[prefillProjectionKey];
2088
+ const prefillProjectionF16Entry = stableTextBoundary
2089
+ ? (
2090
+ deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2091
+ ?? deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2092
+ ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2093
+ )
2094
+ : (
2095
+ deriveQ4PrefillF16AccumKernelEntry(sourcePrefillProjectionEntry)
2096
+ ?? deriveQ4WideTilePrefillF16KernelEntry(sourcePrefillProjectionEntry)
2097
+ ?? deriveQ4PrefillF16KernelEntry(sourcePrefillProjectionEntry)
2098
+ );
2099
+ if (stableTextBoundary && prefillProjectionKey && prefillProjectionF16Entry) {
2100
+ const prefillProjectionF16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_gemma4_f16');
2101
+ result.kernels[prefillProjectionF16Key] = prefillProjectionF16Entry;
2102
+ changed = true;
2103
+ const remapped = remapGemma412BPrefillProjectionEntries(
2104
+ result.prefill,
2105
+ prefillProjectionF16Key,
2106
+ prefillProjectionKey
2107
+ );
2108
+ result.prefill = remapped.entries;
2109
+ changed = changed || remapped.changed;
2110
+ } else {
2111
+ replaceKernelEntry(
2112
+ prefillProjectionKey,
2113
+ prefillProjectionF16Entry
2114
+ );
2115
+ }
2116
2116
 
2117
2117
  const replacePrefillAttentionEntries = (entries) => {
2118
2118
  for (const entry of entries || []) {
@@ -2122,7 +2122,7 @@ export function useGemma431BTextF16Activations(graph, ctx) {
2122
2122
  }
2123
2123
  replaceKernelEntry(
2124
2124
  entry[1],
2125
- deriveF16AttentionKernelEntry(result.kernels[entry[1]])
2125
+ derivePrefillAttentionEntry(result.kernels[entry[1]])
2126
2126
  );
2127
2127
  continue;
2128
2128
  }
@@ -2169,39 +2169,40 @@ export function useGemma431BTextF16Activations(graph, ctx) {
2169
2169
  );
2170
2170
  }
2171
2171
 
2172
+ if (stableTextBoundary) {
2173
+ changed = remapGemma412BStableBoundaryEntries(result, graph) || changed;
2174
+ }
2175
+
2172
2176
  return changed ? result : null;
2173
2177
  }
2174
2178
 
2179
+ export function useGemma4TextF16Activations(graph, ctx) {
2180
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: false });
2181
+ }
2182
+
2183
+ export function useGemma412BTextF16Activations(graph, ctx) {
2184
+ return useGemma4TextF16ActivationsForLane(graph, ctx, { stableTextBoundary: true });
2185
+ }
2186
+
2187
+ export function useGemma431BTextF16Activations(graph, ctx) {
2188
+ return useGemma4TextF16Activations(graph, ctx);
2189
+ }
2190
+
2175
2191
  // =============================================================================
2176
2192
  // Transform: useGemma4Int4PleAf16Activations
2177
2193
  // =============================================================================
2178
2194
 
2179
- /**
2195
+ /*
2180
2196
  * Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
2181
2197
  * weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
2182
- * useGemma431BTextF16Activations: same Q4 weight pack, kernels narrowed to f16
2198
+ * useGemma4TextF16Activations: same Q4 weight pack, kernels narrowed to f16
2183
2199
  * activations, prefill projections promoted from widetile to widetile_f16a,
2184
2200
  * decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
2185
2201
  * counterparts. Apple Metal stays disabled at the capability layer because the
2186
2202
  * fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
2187
2203
  *
2188
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
2189
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2190
- * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
2191
2204
  */
2192
2205
  export function useGemma4Int4PleAf16Activations(graph, ctx) {
2193
- const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
2194
- if (
2195
- modelId !== 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple'
2196
- || ctx.activationDtype !== 'f16'
2197
- || ctx.mathDtype !== 'f16'
2198
- || ctx.accumDtype !== 'f16'
2199
- || ctx.capabilities?.hasF16 !== true
2200
- || ctx.capabilities?.hasSubgroups !== true
2201
- ) {
2202
- return null;
2203
- }
2204
-
2205
2206
  const narrowed = narrowToF16Activations(graph, ctx);
2206
2207
  const result = narrowed ?? cloneGraph(graph);
2207
2208
  let changed = narrowed != null;
@@ -2305,14 +2306,12 @@ export function useGemma4Int4PleAf16Activations(graph, ctx) {
2305
2306
  // Composition
2306
2307
  // =============================================================================
2307
2308
 
2308
- /**
2309
+ /*
2309
2310
  * Compose multiple transforms into a single transform function.
2310
2311
  *
2311
2312
  * Each transform is applied sequentially. If a transform returns null
2312
2313
  * (not applicable), the graph passes through unchanged.
2313
2314
  *
2314
- * @param {Array<(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph | null>} transforms
2315
- * @returns {(graph: import('./execution-graph-transforms.js').ExecutionGraph, ctx: import('./execution-graph-transforms.js').TransformContext) => import('./execution-graph-transforms.js').ExecutionGraph}
2316
2315
  */
2317
2316
  export function composeTransforms(...transforms) {
2318
2317
  return (graph, ctx) => {
@@ -2333,7 +2332,7 @@ export function disableRetainQ4KMaterialization() {
2333
2332
  return null;
2334
2333
  }
2335
2334
 
2336
- /**
2335
+ /*
2337
2336
  * Fail-closed sentinel transform. A capability rule installs this when the
2338
2337
  * matched (modelId, runtime profile) combination is contradictory — for
2339
2338
  * example, an af32 manifest variant paired with a runtime profile that
@@ -2341,9 +2340,6 @@ export function disableRetainQ4KMaterialization() {
2341
2340
  * earlier manifest-binding gate has been bypassed; throwing here keeps the
2342
2341
  * lane-confusion door shut at the capability layer too.
2343
2342
  *
2344
- * @param {import('./execution-graph-transforms.js').ExecutionGraph} _graph
2345
- * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2346
- * @returns {never}
2347
2343
  */
2348
2344
  export function failClosedLaneMismatch(_graph, ctx) {
2349
2345
  const modelId = ctx?.modelId ?? 'unknown';
@@ -2359,7 +2355,6 @@ export function failClosedLaneMismatch(_graph, ctx) {
2359
2355
  // Registry
2360
2356
  // =============================================================================
2361
2357
 
2362
- /** @type {Readonly<Record<string, Function>>} */
2363
2358
  export const TRANSFORMS = Object.freeze({
2364
2359
  narrowToF16Activations,
2365
2360
  removeSubgroups,
@@ -2381,6 +2376,8 @@ export const TRANSFORMS = Object.freeze({
2381
2376
  useQwen36F16Activations,
2382
2377
  useQwenDecodeF16Matmuls,
2383
2378
  useGemma4Int4PleSelectiveF16Decode,
2379
+ useGemma4TextF16Activations,
2380
+ useGemma412BTextF16Activations,
2384
2381
  useGemma431BTextF16Activations,
2385
2382
  useGemma4Int4PleAf16Activations,
2386
2383
  failClosedLaneMismatch,