doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -35,15 +35,20 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
35
35
  ['matmul_gemv_subgroup_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
36
36
  ['matmul_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
37
37
  ['matmul_f16_tiled.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
38
+ ['silu_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
38
39
  ['fused_matmul_q4_multicol_f16.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
39
40
  ['fused_matmul_q4_multicol_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
41
+ ['fused_matmul_q4_batched_f16acc_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
40
42
  ['fused_matmul_q4.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
41
43
  ['fused_matmul_q4_batched.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
42
44
  ['fused_matmul_q4_batched_multicol_shared.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
45
+ ['fused_matmul_q4_widetile_f16a.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
43
46
  ['matmul_gemv_subgroup.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
44
47
  ['matmul_f16w_f32a.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
45
48
  ['matmul_f16w_f32a_tiled.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
46
49
  ['matmul_f32.wgsl', { inputDtype: 'f32', outputDtype: 'f32' }],
50
+ ['gather_f16_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
51
+ ['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
47
52
  ]);
48
53
 
49
54
  /**
@@ -111,6 +116,11 @@ function deriveKernelEntry(base, newFile, newEntry, constants) {
111
116
  return derived;
112
117
  }
113
118
 
119
+ function deriveMappedKernelEntry(base, newFile) {
120
+ const newEntry = newFile === 'matmul_f32.wgsl' ? 'main' : base.entry;
121
+ return deriveKernelEntry(base, newFile, newEntry);
122
+ }
123
+
114
124
  function deriveKernelPrecision(base, newFile) {
115
125
  const precision = base.precision ? { ...base.precision } : {};
116
126
  const precisionPatch = KERNEL_FILE_PRECISION_PATCHES.get(newFile);
@@ -292,6 +302,28 @@ function replacePhaseStepKernelKey(steps, op, replacementKernelKey) {
292
302
  return { steps: nextSteps, changed };
293
303
  }
294
304
 
305
+ function findPhaseStep(steps, op) {
306
+ if (!Array.isArray(steps) || !op) {
307
+ return null;
308
+ }
309
+ for (const entry of steps) {
310
+ if (Array.isArray(entry)) {
311
+ if (entry[0] === op) {
312
+ return entry;
313
+ }
314
+ continue;
315
+ }
316
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
317
+ continue;
318
+ }
319
+ const nested = findPhaseStep(entry.steps, op);
320
+ if (nested) {
321
+ return nested;
322
+ }
323
+ }
324
+ return null;
325
+ }
326
+
295
327
  function deriveKernelEntryWithPrecision(base, precision) {
296
328
  return {
297
329
  ...base,
@@ -307,6 +339,7 @@ const ATTENTION_F16KV_TO_F16_MAP = new Map([
307
339
  ['attention_decode_chunked_f16kv.wgsl', 'attention_decode_chunked_f16.wgsl'],
308
340
  ['attention_small_f16kv.wgsl', 'attention_small_f16.wgsl'],
309
341
  ['attention_streaming_f16kv.wgsl', 'attention_streaming_f16.wgsl'],
342
+ ['attention_head512_f16kv.wgsl', 'attention_head512_f16.wgsl'],
310
343
  ]);
311
344
 
312
345
  function deriveF16AttentionKernelEntry(base) {
@@ -366,6 +399,29 @@ function deriveLmHeadDecodeF16KernelEntry(base) {
366
399
  return null;
367
400
  }
368
401
 
402
+ function deriveDenseDecodeF16KernelEntry(base) {
403
+ if (typeof base?.kernel !== 'string') {
404
+ return null;
405
+ }
406
+ const precision = {
407
+ inputDtype: 'f16',
408
+ outputDtype: 'f16',
409
+ };
410
+ if (base.kernel === 'matmul_gemv_subgroup.wgsl') {
411
+ return {
412
+ ...deriveKernelEntry(base, 'matmul_gemv_subgroup_f16a.wgsl', base.entry ?? 'main'),
413
+ precision: {
414
+ ...(base.precision ?? {}),
415
+ ...precision,
416
+ },
417
+ };
418
+ }
419
+ if (base.kernel === 'matmul_gemv_subgroup_f16a.wgsl') {
420
+ return deriveKernelEntryWithPrecision(base, precision);
421
+ }
422
+ return null;
423
+ }
424
+
369
425
  function deriveQ4DecodeF16KernelEntry(base) {
370
426
  if (typeof base?.kernel !== 'string') {
371
427
  return null;
@@ -400,6 +456,9 @@ function deriveQ4PrefillF16KernelEntry(base) {
400
456
  inputDtype: 'f16',
401
457
  outputDtype: 'f16',
402
458
  };
459
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
460
+ return deriveKernelEntryWithPrecision(base, precision);
461
+ }
403
462
  if (base.kernel.startsWith('fused_matmul_q4_batched')) {
404
463
  return {
405
464
  ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16a.wgsl', 'main_batched_f16a', null),
@@ -412,6 +471,53 @@ function deriveQ4PrefillF16KernelEntry(base) {
412
471
  return null;
413
472
  }
414
473
 
474
+ function deriveQ4WideTilePrefillF16KernelEntry(base) {
475
+ if (typeof base?.kernel !== 'string') {
476
+ return null;
477
+ }
478
+ const precision = {
479
+ inputDtype: 'f16',
480
+ outputDtype: 'f16',
481
+ };
482
+ if (base.kernel === 'fused_matmul_q4_widetile.wgsl') {
483
+ return {
484
+ ...deriveKernelEntry(base, 'fused_matmul_q4_widetile_f16a.wgsl', 'main', null),
485
+ precision: {
486
+ ...(base.precision ?? {}),
487
+ ...precision,
488
+ },
489
+ };
490
+ }
491
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
492
+ return deriveKernelEntryWithPrecision(base, precision);
493
+ }
494
+ return null;
495
+ }
496
+
497
+ function deriveQ4PrefillF16AccumKernelEntry(base) {
498
+ if (typeof base?.kernel !== 'string') {
499
+ return null;
500
+ }
501
+ const precision = {
502
+ inputDtype: 'f16',
503
+ outputDtype: 'f16',
504
+ };
505
+ if (
506
+ base.kernel === 'fused_matmul_q4_widetile.wgsl'
507
+ || base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl'
508
+ || base.kernel.startsWith('fused_matmul_q4_batched')
509
+ ) {
510
+ return {
511
+ ...deriveKernelEntry(base, 'fused_matmul_q4_batched_f16acc_f16a.wgsl', 'main_batched_f16acc_f16a', null),
512
+ precision: {
513
+ ...(base.precision ?? {}),
514
+ ...precision,
515
+ },
516
+ };
517
+ }
518
+ return null;
519
+ }
520
+
415
521
  function deriveQ4DecodeF32ActivationKernelEntry(base) {
416
522
  if (typeof base?.kernel !== 'string') {
417
523
  return null;
@@ -445,9 +551,16 @@ function deriveQ4PrefillF32ActivationKernelEntry(base) {
445
551
  if (
446
552
  base.kernel !== 'fused_matmul_q4_batched_f16.wgsl'
447
553
  && base.kernel !== 'fused_matmul_q4_batched_f16a.wgsl'
554
+ && base.kernel !== 'fused_matmul_q4_widetile_f16a.wgsl'
448
555
  ) {
449
556
  return null;
450
557
  }
558
+ if (base.kernel === 'fused_matmul_q4_widetile_f16a.wgsl') {
559
+ return deriveKernelEntryWithPrecision(
560
+ deriveKernelEntry(base, 'fused_matmul_q4_widetile.wgsl', 'main', null),
561
+ { inputDtype: 'f32', outputDtype: 'f32' }
562
+ );
563
+ }
451
564
  return deriveKernelEntryWithPrecision(
452
565
  deriveKernelEntry(base, 'fused_matmul_q4_batched.wgsl', 'main_batched', null),
453
566
  { inputDtype: 'f32', outputDtype: 'f32' }
@@ -582,8 +695,11 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
582
695
  ['rope_f16.wgsl', 'rope.wgsl'],
583
696
  ['residual_f16.wgsl', 'residual.wgsl'],
584
697
  ['gelu_f16.wgsl', 'gelu.wgsl'],
698
+ ['silu_f16.wgsl', 'silu.wgsl'],
585
699
  ['sample_f16.wgsl', 'sample.wgsl'],
586
700
  ['gather_f16.wgsl', 'gather.wgsl'],
701
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
702
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
587
703
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
588
704
  ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
589
705
  ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
@@ -591,6 +707,7 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
591
707
  ['attention_decode_chunked_f16.wgsl', 'attention_decode_chunked_f16kv.wgsl'],
592
708
  ['attention_small_f16.wgsl', 'attention_small_f16kv.wgsl'],
593
709
  ['attention_streaming_f16.wgsl', 'attention_streaming_f16kv.wgsl'],
710
+ ['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
594
711
  ]);
595
712
 
596
713
  /**
@@ -605,6 +722,11 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
605
722
  const F32_TO_F16_ACTIVATION_MAP = new Map(
606
723
  Array.from(F16_TO_F32_ACTIVATION_MAP.entries(), ([from, to]) => [to, from])
607
724
  );
725
+ F32_TO_F16_ACTIVATION_MAP.set('gather.wgsl', 'gather_f16.wgsl');
726
+ F32_TO_F16_ACTIVATION_MAP.set('attention_head256_f16kv.wgsl', 'attention_small_f16.wgsl');
727
+ // head512 pure-f16 prefill is model-scoped below; keep generic Gemma 4 E2B
728
+ // f16 requests fail-closed until that path has its own evidence.
729
+ F32_TO_F16_ACTIVATION_MAP.delete('attention_head512_f16kv.wgsl');
608
730
 
609
731
  function hasExplicitF32ActivationContract(entry) {
610
732
  const precision = entry?.precision;
@@ -627,17 +749,20 @@ const F16_TO_F32_CORRECTNESS_FALLBACK_MAP = new Map([
627
749
  ['rope_f16.wgsl', 'rope.wgsl'],
628
750
  ['residual_f16.wgsl', 'residual.wgsl'],
629
751
  ['gelu_f16.wgsl', 'gelu.wgsl'],
752
+ ['silu_f16.wgsl', 'silu.wgsl'],
630
753
  ['sample_f16.wgsl', 'sample.wgsl'],
631
754
  ['gather_f16.wgsl', 'gather.wgsl'],
755
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
756
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
632
757
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_gemv_subgroup.wgsl'],
633
758
  ['matmul_f16.wgsl', 'matmul_f16w_f32a.wgsl'],
634
759
  ['matmul_f16_tiled.wgsl', 'matmul_f16w_f32a_tiled.wgsl'],
635
- ['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
636
- ['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
760
+ ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
761
+ ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
637
762
  ['attention_small_f16.wgsl', 'attention_small.wgsl'],
638
763
  ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
639
- ['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
640
- ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
764
+ ['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
765
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
641
766
  ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
642
767
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
643
768
  ]);
@@ -654,8 +779,11 @@ const FULL_F32_SHADER_MAP = new Map([
654
779
  ['rope_f16.wgsl', 'rope.wgsl'],
655
780
  ['residual_f16.wgsl', 'residual.wgsl'],
656
781
  ['gelu_f16.wgsl', 'gelu.wgsl'],
782
+ ['silu_f16.wgsl', 'silu.wgsl'],
657
783
  ['sample_f16.wgsl', 'sample.wgsl'],
658
784
  ['gather_f16.wgsl', 'gather.wgsl'],
785
+ ['gather_f16_f16_out.wgsl', 'gather.wgsl'],
786
+ ['gather_f16_vec4_f16_out.wgsl', 'gather.wgsl'],
659
787
  // f16-activation matmul → f32
660
788
  ['matmul_gemv_subgroup_f16a.wgsl', 'matmul_f32.wgsl'],
661
789
  ['matmul_f16.wgsl', 'matmul_f32.wgsl'],
@@ -665,13 +793,13 @@ const FULL_F32_SHADER_MAP = new Map([
665
793
  ['matmul_f16w_f32a.wgsl', 'matmul_f32.wgsl'],
666
794
  ['matmul_f16w_f32a_tiled.wgsl', 'matmul_f32.wgsl'],
667
795
  // f16-activation attention → f32
668
- ['attention_decode_online_f16.wgsl', 'attention_decode.wgsl'],
669
- ['attention_decode_chunked_f16.wgsl', 'attention_decode.wgsl'],
796
+ ['attention_decode_online_f16.wgsl', 'attention_streaming.wgsl'],
797
+ ['attention_decode_chunked_f16.wgsl', 'attention_streaming.wgsl'],
670
798
  ['attention_small_f16.wgsl', 'attention_small.wgsl'],
671
799
  ['attention_streaming_f16.wgsl', 'attention_streaming.wgsl'],
672
800
  // f16kv attention (f32 Q, f16 KV) → f32
673
- ['attention_decode_online_f16kv.wgsl', 'attention_decode.wgsl'],
674
- ['attention_decode_chunked_f16kv.wgsl', 'attention_decode.wgsl'],
801
+ ['attention_decode_online_f16kv.wgsl', 'attention_streaming.wgsl'],
802
+ ['attention_decode_chunked_f16kv.wgsl', 'attention_streaming.wgsl'],
675
803
  ['attention_small_f16kv.wgsl', 'attention_small.wgsl'],
676
804
  ['attention_streaming_f16kv.wgsl', 'attention_streaming.wgsl'],
677
805
  ['attention_head256_f16kv.wgsl', 'attention_small.wgsl'],
@@ -721,7 +849,7 @@ export function widenToF32Activations(graph, ctx) {
721
849
  }
722
850
  const replacement = shaderMap.get(entry.kernel);
723
851
  if (replacement !== undefined) {
724
- result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
852
+ result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
725
853
  }
726
854
  }
727
855
 
@@ -758,7 +886,7 @@ export function widenToF32CorrectnessFallback(graph, ctx) {
758
886
  }
759
887
  const replacement = F16_TO_F32_CORRECTNESS_FALLBACK_MAP.get(entry.kernel);
760
888
  if (replacement !== undefined) {
761
- result.kernels[key] = deriveKernelEntry(entry, replacement, entry.entry);
889
+ result.kernels[key] = deriveMappedKernelEntry(entry, replacement);
762
890
  }
763
891
  }
764
892
  return result;
@@ -812,6 +940,26 @@ const PREFILL_ATTENTION_PAIRS = new Map([
812
940
  ['attention_small_f16.wgsl', 'attention_streaming_f16.wgsl'],
813
941
  ]);
814
942
 
943
+ function graphUsesKernelKeyInPrefill(graph, kernelKey) {
944
+ for (const entry of graph.prefill || []) {
945
+ if (Array.isArray(entry)) {
946
+ if (entry[1] === kernelKey) {
947
+ return true;
948
+ }
949
+ continue;
950
+ }
951
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.steps)) {
952
+ continue;
953
+ }
954
+ for (const step of entry.steps) {
955
+ if (Array.isArray(step) && step[1] === kernelKey) {
956
+ return true;
957
+ }
958
+ }
959
+ }
960
+ return false;
961
+ }
962
+
815
963
  /**
816
964
  * Swap prefill attention kernel between streaming and small variants.
817
965
  *
@@ -847,8 +995,7 @@ export function swapPrefillAttention(graph, ctx, opts) {
847
995
  }
848
996
 
849
997
  if (target !== undefined) {
850
- // Verify this kernel is actually used in a prefill step
851
- const usedInPrefill = (graph.prefill || []).some((step) => step[1] === key);
998
+ const usedInPrefill = graphUsesKernelKeyInPrefill(graph, key);
852
999
  if (usedInPrefill) {
853
1000
  result.kernels[key] = deriveKernelEntry(entry, target, entry.entry);
854
1001
  changed = true;
@@ -863,6 +1010,20 @@ export function swapPrefillAttention(graph, ctx, opts) {
863
1010
  // Transform: useHead256PrefillAttention
864
1011
  // =============================================================================
865
1012
 
1013
+ /**
1014
+ * Promote small-tile prefill attention onto the fixed 256-dim shared-block kernel.
1015
+ *
1016
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1017
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1018
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1019
+ */
1020
+ export function useHead256SmallPrefillAttention(graph, ctx) {
1021
+ return swapPrefillAttention(graph, ctx, {
1022
+ from: 'attention_small_f16kv.wgsl',
1023
+ to: 'attention_head256_f16kv.wgsl',
1024
+ });
1025
+ }
1026
+
866
1027
  /**
867
1028
  * Promote prefill attention onto the fixed 256-dim shared-block kernel.
868
1029
  *
@@ -871,16 +1032,25 @@ export function swapPrefillAttention(graph, ctx, opts) {
871
1032
  * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
872
1033
  */
873
1034
  export function useHead256PrefillAttention(graph, ctx) {
874
- return (
875
- swapPrefillAttention(graph, ctx, {
876
- from: 'attention_small_f16kv.wgsl',
877
- to: 'attention_head256_f16kv.wgsl',
878
- })
879
- || swapPrefillAttention(graph, ctx, {
880
- from: 'attention_streaming_f16kv.wgsl',
881
- to: 'attention_head256_f16kv.wgsl',
882
- })
883
- );
1035
+ let current = graph;
1036
+ let changed = false;
1037
+
1038
+ const smallRemap = useHead256SmallPrefillAttention(current, ctx);
1039
+ if (smallRemap) {
1040
+ current = smallRemap;
1041
+ changed = true;
1042
+ }
1043
+
1044
+ const streamingRemap = swapPrefillAttention(current, ctx, {
1045
+ from: 'attention_streaming_f16kv.wgsl',
1046
+ to: 'attention_head256_f16kv.wgsl',
1047
+ });
1048
+ if (streamingRemap) {
1049
+ current = streamingRemap;
1050
+ changed = true;
1051
+ }
1052
+
1053
+ return changed ? current : null;
884
1054
  }
885
1055
 
886
1056
  // =============================================================================
@@ -1561,14 +1731,16 @@ export function useQwenDecodeF16Matmuls(graph, ctx) {
1561
1731
  // =============================================================================
1562
1732
 
1563
1733
  /**
1564
- * Promote the Qwen 3.5 0.8B execution graph onto its fast f16 primary lane
1565
- * when the runtime explicitly requests f16 activations.
1734
+ * Promote the Qwen 3.5 0.8B execution graph onto its selective f16 primary
1735
+ * lane when the runtime explicitly requests f16 activations.
1566
1736
  *
1567
- * This transform narrows the hot matmul path while keeping `o_proj` on the
1568
- * stable manifest-owned f32-output kernel. The residual stream feeds directly
1569
- * into the next RMSNorm, and the Qwen 3.5 0.8B promoted f16 lane becomes
1570
- * numerically unstable when `o_proj` writes f16 there (observed first at the
1571
- * first full-attention block's post-attention RMSNorm).
1737
+ * This transform narrows the decode projection and LM-head path while keeping
1738
+ * `o_proj` on the stable manifest-owned f32-output kernel. The prefill path is
1739
+ * already authored in the manifest on WideTile/head256 kernels and remains
1740
+ * manifest-owned. The residual stream feeds directly into the next RMSNorm, and
1741
+ * the Qwen 3.5 0.8B promoted f16 lane becomes numerically unstable when
1742
+ * `o_proj` writes f16 there (observed first at the first full-attention block's
1743
+ * post-attention RMSNorm).
1572
1744
  *
1573
1745
  * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1574
1746
  * @param {import('./execution-graph-transforms.js').TransformContext} ctx
@@ -1672,6 +1844,463 @@ export function useQwenF16PrimaryMatmuls(graph, ctx) {
1672
1844
  return changed ? result : null;
1673
1845
  }
1674
1846
 
1847
+ // =============================================================================
1848
+ // Transform: useQwen36F16Activations
1849
+ // =============================================================================
1850
+
1851
+ /**
1852
+ * Promote the Qwen 3.6 27B Q4K graph onto its additive all-f16 sibling lane.
1853
+ *
1854
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1855
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1856
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1857
+ */
1858
+ export function useQwen36F16Activations(graph, ctx) {
1859
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1860
+ if (
1861
+ modelId !== 'qwen-3-6-27b-q4k-eaf16'
1862
+ || ctx.activationDtype !== 'f16'
1863
+ || ctx.mathDtype !== 'f16'
1864
+ || ctx.accumDtype !== 'f16'
1865
+ || ctx.capabilities?.hasF16 !== true
1866
+ || ctx.capabilities?.hasSubgroups !== true
1867
+ ) {
1868
+ return null;
1869
+ }
1870
+
1871
+ const narrowed = narrowToF16Activations(graph, ctx);
1872
+ const result = narrowed ?? cloneGraph(graph);
1873
+ let changed = narrowed != null;
1874
+
1875
+ const replaceKernelEntry = (key, entry) => {
1876
+ if (!key || !entry) {
1877
+ return;
1878
+ }
1879
+ result.kernels[key] = entry;
1880
+ changed = true;
1881
+ };
1882
+ const replaceOps = (phaseName, ops, kernelKey) => {
1883
+ for (const op of ops) {
1884
+ const phaseResult = replacePhaseStepKernelKey(result[phaseName], op, kernelKey);
1885
+ if (phaseResult.changed) {
1886
+ result[phaseName] = phaseResult.steps;
1887
+ changed = true;
1888
+ }
1889
+ }
1890
+ };
1891
+
1892
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
1893
+ const embedKey = embedStep?.[1] ?? null;
1894
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
1895
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
1896
+ replaceKernelEntry(
1897
+ embedKey,
1898
+ deriveKernelEntryWithPrecision(
1899
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
1900
+ { inputDtype: 'f16', outputDtype: 'f16' }
1901
+ )
1902
+ );
1903
+ }
1904
+
1905
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
1906
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
1907
+ const decodeProjectionEntry = deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey]);
1908
+ if (decodeProjectionEntry) {
1909
+ const f16Key = deriveKernelKey(result.kernels, decodeProjectionKey, '_qwen36_f16');
1910
+ result.kernels[f16Key] = decodeProjectionEntry;
1911
+ replaceOps('decode', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1912
+ }
1913
+
1914
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
1915
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
1916
+ const prefillProjectionEntry =
1917
+ deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
1918
+ ?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
1919
+ ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey]);
1920
+ if (prefillProjectionEntry) {
1921
+ const f16Key = deriveKernelKey(result.kernels, prefillProjectionKey, '_qwen36_f16');
1922
+ result.kernels[f16Key] = prefillProjectionEntry;
1923
+ replaceOps('prefill', ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'], f16Key);
1924
+ }
1925
+
1926
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
1927
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
1928
+ replaceKernelEntry(
1929
+ lmHeadKey,
1930
+ deriveQ4DecodeF16KernelEntry(result.kernels[lmHeadKey])
1931
+ );
1932
+
1933
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
1934
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
1935
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
1936
+ if (
1937
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
1938
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
1939
+ ) {
1940
+ replaceKernelEntry(
1941
+ lmHeadPrefillKey,
1942
+ deriveKernelEntryWithPrecision(
1943
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
1944
+ { inputDtype: 'f16', outputDtype: 'f16' }
1945
+ )
1946
+ );
1947
+ }
1948
+
1949
+ return changed ? result : null;
1950
+ }
1951
+
1952
+ // =============================================================================
1953
+ // Transform: useGemma4Int4PleSelectiveF16Decode
1954
+ // =============================================================================
1955
+
1956
+ /**
1957
+ * Promote only Gemma 4 E2B INT4 PLE decode Q/K/V and online attention onto
1958
+ * explicit f16 kernels. Prefill remains on manifest-owned f16kv fixed-head
1959
+ * attention because the repository does not currently have pure-f16 head256 or
1960
+ * head512 prefill kernels.
1961
+ *
1962
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
1963
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
1964
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
1965
+ */
1966
+ export function useGemma4Int4PleSelectiveF16Decode(graph, ctx) {
1967
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
1968
+ if (
1969
+ modelId !== 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple'
1970
+ || ctx.activationDtype !== 'f16'
1971
+ || ctx.capabilities?.hasF16 !== true
1972
+ || ctx.capabilities?.hasSubgroups !== true
1973
+ ) {
1974
+ return null;
1975
+ }
1976
+
1977
+ const result = cloneGraph(graph);
1978
+ let changed = false;
1979
+
1980
+ const decodeProjectionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'q_proj');
1981
+ const decodeProjectionKernel = decodeProjectionStep ? result.kernels[decodeProjectionStep[1]] : null;
1982
+ const decodeProjectionEntry = deriveDenseDecodeF16KernelEntry(decodeProjectionKernel);
1983
+ if (decodeProjectionStep && decodeProjectionEntry) {
1984
+ const decodeProjectionKey = deriveKernelKey(result.kernels, decodeProjectionStep[1], '_gemma4_f16');
1985
+ result.kernels[decodeProjectionKey] = decodeProjectionEntry;
1986
+ for (const op of ['q_proj', 'k_proj', 'v_proj']) {
1987
+ const phaseResult = replacePhaseStepKernelKey(result.decode, op, decodeProjectionKey);
1988
+ if (phaseResult.changed) {
1989
+ result.decode = phaseResult.steps;
1990
+ changed = true;
1991
+ }
1992
+ }
1993
+ }
1994
+
1995
+ for (const op of ['rope_q', 'rope_k']) {
1996
+ const step = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === op);
1997
+ const kernelKey = step?.[1];
1998
+ const kernelEntry = kernelKey ? result.kernels[kernelKey] : null;
1999
+ const replacement = kernelEntry ? F32_TO_F16_ACTIVATION_MAP.get(kernelEntry.kernel) : null;
2000
+ if (!step || !kernelKey || !replacement) {
2001
+ continue;
2002
+ }
2003
+ const ropeKey = deriveKernelKey(result.kernels, kernelKey, '_gemma4_f16');
2004
+ result.kernels[ropeKey] = deriveKernelEntryWithPrecision(
2005
+ deriveKernelEntry(kernelEntry, replacement, kernelEntry.entry),
2006
+ { inputDtype: 'f16', outputDtype: 'f16' }
2007
+ );
2008
+ const phaseResult = replacePhaseStepKernelKey(result.decode, op, ropeKey);
2009
+ if (phaseResult.changed) {
2010
+ result.decode = phaseResult.steps;
2011
+ changed = true;
2012
+ }
2013
+ }
2014
+
2015
+ const attentionStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'attention');
2016
+ const attentionKernel = attentionStep ? result.kernels[attentionStep[1]] : null;
2017
+ const attentionEntry = deriveF16AttentionKernelEntry(attentionKernel);
2018
+ if (attentionStep && attentionEntry) {
2019
+ const attentionKey = deriveKernelKey(result.kernels, attentionStep[1], '_gemma4_f16');
2020
+ result.kernels[attentionKey] = attentionEntry;
2021
+ const phaseResult = replacePhaseStepKernelKey(result.decode, 'attention', attentionKey);
2022
+ if (phaseResult.changed) {
2023
+ result.decode = phaseResult.steps;
2024
+ changed = true;
2025
+ }
2026
+ }
2027
+
2028
+ const oProjStep = (result.decode || []).find((entry) => Array.isArray(entry) && entry[0] === 'o_proj');
2029
+ const oProjKernel = oProjStep ? result.kernels[oProjStep[1]] : null;
2030
+ if (oProjStep && oProjKernel) {
2031
+ const oProjKey = deriveKernelKey(result.kernels, oProjStep[1], '_gemma4_f32_boundary');
2032
+ result.kernels[oProjKey] = deriveKernelEntryWithPrecision(oProjKernel, {
2033
+ inputDtype: 'f32',
2034
+ outputDtype: 'f32',
2035
+ });
2036
+ const phaseResult = replacePhaseStepKernelKey(result.decode, 'o_proj', oProjKey);
2037
+ if (phaseResult.changed) {
2038
+ result.decode = phaseResult.steps;
2039
+ changed = true;
2040
+ }
2041
+ }
2042
+
2043
+ return changed ? result : null;
2044
+ }
2045
+
2046
+ // =============================================================================
2047
+ // Transform: useGemma431BTextF16Activations
2048
+ // =============================================================================
2049
+
2050
+ /**
2051
+ * Promote the Gemma 4 31B text Q4K graph onto an experimental all-f16 request
2052
+ * lane. This keeps the same Q4K weight pack but remaps execution-v1 kernels so
2053
+ * Q4 projections, attention, residual/norm/activation utilities, lm_head, and
2054
+ * sampling consume and produce f16 activations where a matching kernel exists.
2055
+ *
2056
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
2057
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2058
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
2059
+ */
2060
+ export function useGemma431BTextF16Activations(graph, ctx) {
2061
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
2062
+ if (
2063
+ ![
2064
+ 'gemma-4-31b-it-text-q4k-ehf16-af32',
2065
+ 'gemma-4-31b-it-text-q4k-ehf16-af16',
2066
+ ].includes(modelId)
2067
+ || ctx.activationDtype !== 'f16'
2068
+ || ctx.mathDtype !== 'f16'
2069
+ || ctx.accumDtype !== 'f16'
2070
+ || ctx.capabilities?.hasF16 !== true
2071
+ || ctx.capabilities?.hasSubgroups !== true
2072
+ ) {
2073
+ return null;
2074
+ }
2075
+
2076
+ const narrowed = narrowToF16Activations(graph, ctx);
2077
+ const result = narrowed ?? cloneGraph(graph);
2078
+ let changed = narrowed != null;
2079
+
2080
+ const replaceKernelEntry = (key, entry) => {
2081
+ if (!key || !entry) {
2082
+ return;
2083
+ }
2084
+ result.kernels[key] = entry;
2085
+ changed = true;
2086
+ };
2087
+
2088
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
2089
+ const embedKey = embedStep?.[1] ?? null;
2090
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
2091
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
2092
+ replaceKernelEntry(
2093
+ embedKey,
2094
+ deriveKernelEntryWithPrecision(
2095
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2096
+ { inputDtype: 'f16', outputDtype: 'f16' }
2097
+ )
2098
+ );
2099
+ }
2100
+
2101
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2102
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2103
+ replaceKernelEntry(
2104
+ decodeProjectionKey,
2105
+ deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2106
+ );
2107
+
2108
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2109
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2110
+ replaceKernelEntry(
2111
+ prefillProjectionKey,
2112
+ deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
2113
+ ?? deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2114
+ ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2115
+ );
2116
+
2117
+ const replacePrefillAttentionEntries = (entries) => {
2118
+ for (const entry of entries || []) {
2119
+ if (Array.isArray(entry)) {
2120
+ if (entry[0] !== 'attention') {
2121
+ continue;
2122
+ }
2123
+ replaceKernelEntry(
2124
+ entry[1],
2125
+ deriveF16AttentionKernelEntry(result.kernels[entry[1]])
2126
+ );
2127
+ continue;
2128
+ }
2129
+ if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2130
+ replacePrefillAttentionEntries(entry.steps);
2131
+ }
2132
+ }
2133
+ };
2134
+ replacePrefillAttentionEntries(result.prefill);
2135
+
2136
+ const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2137
+ const finalNormKey = finalNormStep?.[1] ?? null;
2138
+ const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2139
+ if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2140
+ replaceKernelEntry(
2141
+ finalNormKey,
2142
+ deriveKernelEntryWithPrecision(
2143
+ deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2144
+ { inputDtype: 'f16', outputDtype: 'f16' }
2145
+ )
2146
+ );
2147
+ }
2148
+
2149
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2150
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
2151
+ replaceKernelEntry(
2152
+ lmHeadKey,
2153
+ deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2154
+ );
2155
+
2156
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2157
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2158
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2159
+ if (
2160
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2161
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2162
+ ) {
2163
+ replaceKernelEntry(
2164
+ lmHeadPrefillKey,
2165
+ deriveKernelEntryWithPrecision(
2166
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2167
+ { inputDtype: 'f16', outputDtype: 'f16' }
2168
+ )
2169
+ );
2170
+ }
2171
+
2172
+ return changed ? result : null;
2173
+ }
2174
+
2175
+ // =============================================================================
2176
+ // Transform: useGemma4Int4PleAf16Activations
2177
+ // =============================================================================
2178
+
2179
+ /**
2180
+ * Promote the Gemma 4 E2B INT4-PLE Q4K graph onto the all-f16 lane via the
2181
+ * weights-ref sibling manifest gemma-4-e2b-it-q4k-ehf16-af16-int4ple. Mirrors
2182
+ * useGemma431BTextF16Activations: same Q4 weight pack, kernels narrowed to f16
2183
+ * activations, prefill projections promoted from widetile to widetile_f16a,
2184
+ * decode projections to multicol_f16a, lm_head/sample/final_norm to their f16
2185
+ * counterparts. Apple Metal stays disabled at the capability layer because the
2186
+ * fused-q4k+f16 kernel pool produces NaN at L0.ffn_down on metal-3.
2187
+ *
2188
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} graph
2189
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2190
+ * @returns {import('./execution-graph-transforms.js').ExecutionGraph | null}
2191
+ */
2192
+ export function useGemma4Int4PleAf16Activations(graph, ctx) {
2193
+ const modelId = typeof ctx.modelId === 'string' ? ctx.modelId.trim() : '';
2194
+ if (
2195
+ modelId !== 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple'
2196
+ || ctx.activationDtype !== 'f16'
2197
+ || ctx.mathDtype !== 'f16'
2198
+ || ctx.accumDtype !== 'f16'
2199
+ || ctx.capabilities?.hasF16 !== true
2200
+ || ctx.capabilities?.hasSubgroups !== true
2201
+ ) {
2202
+ return null;
2203
+ }
2204
+
2205
+ const narrowed = narrowToF16Activations(graph, ctx);
2206
+ const result = narrowed ?? cloneGraph(graph);
2207
+ let changed = narrowed != null;
2208
+
2209
+ const replaceKernelEntry = (key, entry) => {
2210
+ if (!key || !entry) {
2211
+ return;
2212
+ }
2213
+ result.kernels[key] = entry;
2214
+ changed = true;
2215
+ };
2216
+
2217
+ const embedStep = findPhaseStep(result.preLayer, 'embed');
2218
+ const embedKey = embedStep?.[1] ?? null;
2219
+ const embedEntry = embedKey ? result.kernels[embedKey] : null;
2220
+ if (embedEntry?.kernel === 'gather_f16.wgsl') {
2221
+ replaceKernelEntry(
2222
+ embedKey,
2223
+ deriveKernelEntryWithPrecision(
2224
+ deriveKernelEntry(embedEntry, 'gather_f16_vec4_f16_out.wgsl', 'gather_vec4_f16_out'),
2225
+ { inputDtype: 'f16', outputDtype: 'f16' }
2226
+ )
2227
+ );
2228
+ }
2229
+
2230
+ const decodeProjectionStep = findPhaseStep(result.decode, 'q_proj');
2231
+ const decodeProjectionKey = decodeProjectionStep?.[1] ?? null;
2232
+ replaceKernelEntry(
2233
+ decodeProjectionKey,
2234
+ deriveQ4DecodeF16KernelEntry(result.kernels[decodeProjectionKey])
2235
+ );
2236
+
2237
+ const prefillProjectionStep = findPhaseStep(result.prefill, 'q_proj');
2238
+ const prefillProjectionKey = prefillProjectionStep?.[1] ?? null;
2239
+ replaceKernelEntry(
2240
+ prefillProjectionKey,
2241
+ deriveQ4WideTilePrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2242
+ ?? deriveQ4PrefillF16AccumKernelEntry(result.kernels[prefillProjectionKey])
2243
+ ?? deriveQ4PrefillF16KernelEntry(result.kernels[prefillProjectionKey])
2244
+ );
2245
+
2246
+ const replacePrefillAttentionEntries = (entries) => {
2247
+ for (const entry of entries || []) {
2248
+ if (Array.isArray(entry)) {
2249
+ if (entry[0] !== 'attention') {
2250
+ continue;
2251
+ }
2252
+ replaceKernelEntry(
2253
+ entry[1],
2254
+ deriveF16AttentionKernelEntry(result.kernels[entry[1]])
2255
+ );
2256
+ continue;
2257
+ }
2258
+ if (entry && typeof entry === 'object' && Array.isArray(entry.steps)) {
2259
+ replacePrefillAttentionEntries(entry.steps);
2260
+ }
2261
+ }
2262
+ };
2263
+ replacePrefillAttentionEntries(result.prefill);
2264
+
2265
+ const finalNormStep = findPhaseStep(result.postLayer, 'final_norm');
2266
+ const finalNormKey = finalNormStep?.[1] ?? null;
2267
+ const finalNormEntry = finalNormKey ? result.kernels[finalNormKey] : null;
2268
+ if (finalNormEntry?.kernel === 'rmsnorm.wgsl') {
2269
+ replaceKernelEntry(
2270
+ finalNormKey,
2271
+ deriveKernelEntryWithPrecision(
2272
+ deriveKernelEntry(finalNormEntry, 'rmsnorm_f16.wgsl', finalNormEntry.entry),
2273
+ { inputDtype: 'f16', outputDtype: 'f16' }
2274
+ )
2275
+ );
2276
+ }
2277
+
2278
+ const lmHeadStep = findPhaseStep(result.postLayer, 'lm_head');
2279
+ const lmHeadKey = lmHeadStep?.[1] ?? null;
2280
+ replaceKernelEntry(
2281
+ lmHeadKey,
2282
+ deriveLmHeadDecodeF16KernelEntry(result.kernels[lmHeadKey])
2283
+ );
2284
+
2285
+ const lmHeadPrefillStep = findPhaseStep(result.postLayer, 'lm_head_prefill');
2286
+ const lmHeadPrefillKey = lmHeadPrefillStep?.[1] ?? null;
2287
+ const lmHeadPrefillEntry = lmHeadPrefillKey ? result.kernels[lmHeadPrefillKey] : null;
2288
+ if (
2289
+ lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a.wgsl'
2290
+ || lmHeadPrefillEntry?.kernel === 'matmul_f16w_f32a_tiled.wgsl'
2291
+ ) {
2292
+ replaceKernelEntry(
2293
+ lmHeadPrefillKey,
2294
+ deriveKernelEntryWithPrecision(
2295
+ deriveKernelEntry(lmHeadPrefillEntry, 'matmul_f16_tiled.wgsl', 'main'),
2296
+ { inputDtype: 'f16', outputDtype: 'f16' }
2297
+ )
2298
+ );
2299
+ }
2300
+
2301
+ return changed ? result : null;
2302
+ }
2303
+
1675
2304
  // =============================================================================
1676
2305
  // Composition
1677
2306
  // =============================================================================
@@ -1698,6 +2327,34 @@ export function composeTransforms(...transforms) {
1698
2327
  };
1699
2328
  }
1700
2329
 
2330
+ // Session-only capability transform marker. The execution-v1 compiler applies
2331
+ // the matching runtime session patch; the graph itself is intentionally stable.
2332
+ export function disableRetainQ4KMaterialization() {
2333
+ return null;
2334
+ }
2335
+
2336
+ /**
2337
+ * Fail-closed sentinel transform. A capability rule installs this when the
2338
+ * matched (modelId, runtime profile) combination is contradictory — for
2339
+ * example, an af32 manifest variant paired with a runtime profile that
2340
+ * demands f16 activations. The matcher reaches this transform only when the
2341
+ * earlier manifest-binding gate has been bypassed; throwing here keeps the
2342
+ * lane-confusion door shut at the capability layer too.
2343
+ *
2344
+ * @param {import('./execution-graph-transforms.js').ExecutionGraph} _graph
2345
+ * @param {import('./execution-graph-transforms.js').TransformContext} ctx
2346
+ * @returns {never}
2347
+ */
2348
+ export function failClosedLaneMismatch(_graph, ctx) {
2349
+ const modelId = ctx?.modelId ?? 'unknown';
2350
+ const activationDtype = ctx?.activationDtype ?? 'unknown';
2351
+ throw new Error(
2352
+ `Capability resolver: lane mismatch for "${modelId}" (activationDtype=${activationDtype}). ` +
2353
+ 'The manifest variant tag is the lane identity — load the manifest variant ' +
2354
+ 'whose compute lane matches the runtime profile.'
2355
+ );
2356
+ }
2357
+
1701
2358
  // =============================================================================
1702
2359
  // Registry
1703
2360
  // =============================================================================
@@ -1709,6 +2366,7 @@ export const TRANSFORMS = Object.freeze({
1709
2366
  widenToF32Activations,
1710
2367
  widenToF32CorrectnessFallback,
1711
2368
  swapPrefillAttention,
2369
+ useHead256SmallPrefillAttention,
1712
2370
  useHead256PrefillAttention,
1713
2371
  widenProjectionWeightsToF32,
1714
2372
  remapDenseQ4KPrefillToQ4Native,
@@ -1718,7 +2376,13 @@ export const TRANSFORMS = Object.freeze({
1718
2376
  remapQ4KDecodeAttentionToGemv,
1719
2377
  remapQ4KDecodeAttentionToFusedQ4KGemv,
1720
2378
  remapQ4KDecodeFFNToGemv,
2379
+ disableRetainQ4KMaterialization,
1721
2380
  useQwenF16PrimaryMatmuls,
2381
+ useQwen36F16Activations,
1722
2382
  useQwenDecodeF16Matmuls,
2383
+ useGemma4Int4PleSelectiveF16Decode,
2384
+ useGemma431BTextF16Activations,
2385
+ useGemma4Int4PleAf16Activations,
2386
+ failClosedLaneMismatch,
1723
2387
  composeTransforms,
1724
2388
  });