doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -18,12 +18,20 @@ import { composeTransforms } from '../../../config/transforms/execution-graph-tr
18
18
  import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
19
19
  import { log } from '../../../debug/index.js';
20
20
 
21
+ const SESSION_CAPABILITY_TRANSFORMS = new Set([
22
+ 'disableRetainQ4KMaterialization',
23
+ ]);
24
+
21
25
  export function hasExecutionV1(manifestInference) {
22
26
  return manifestInference?.schema === EXECUTION_V1_SCHEMA_ID
23
27
  && manifestInference?.execution
24
28
  && typeof manifestInference.execution.kernels === 'object';
25
29
  }
26
30
 
31
+ function isSessionCapabilityTransform(name) {
32
+ return SESSION_CAPABILITY_TRANSFORMS.has(name);
33
+ }
34
+
27
35
  function mergeExecutionV1Session(manifestSession, runtimeSession) {
28
36
  return mergeRuntimeValues(
29
37
  manifestSession ?? {},
@@ -386,6 +394,8 @@ export function compileExecutionV1(options = {}) {
386
394
  }
387
395
 
388
396
  const activationDtype = session.compute.defaults.activationDtype;
397
+ const mathDtype = session.compute.defaults.mathDtype ?? null;
398
+ const accumDtype = session.compute.defaults.accumDtype ?? null;
389
399
  const kvDtype = session?.kvcache?.kvDtype ?? activationDtype;
390
400
  const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
391
401
  const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
@@ -404,10 +414,13 @@ export function compileExecutionV1(options = {}) {
404
414
  if (capabilities) {
405
415
  const graphContext = {
406
416
  activationDtype,
417
+ mathDtype,
418
+ accumDtype,
407
419
  kvDtype,
408
420
  headDim,
409
421
  modelId,
410
422
  layerTypes,
423
+ retainQ4KMaterialization: session?.retainQ4KMaterialization === true,
411
424
  ...summarizeExecutionGraphContext(execution),
412
425
  };
413
426
  const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
@@ -424,22 +437,35 @@ export function compileExecutionV1(options = {}) {
424
437
  'Use capability-aware remap for manifest-owned execution graphs or choose a compatible runtime.'
425
438
  );
426
439
  }
427
- const composed = composeTransforms(...resolved.transforms);
428
- const transformed = composed(execution, {
429
- capabilities,
430
- platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
431
- activationDtype,
432
- kvDtype,
433
- modelId,
434
- layerTypes,
440
+ const graphTransforms = resolved.transforms.filter((_, index) => {
441
+ return !isSessionCapabilityTransform(resolved.names[index]);
435
442
  });
436
- if (transformed !== execution) {
437
- execution = transformed;
438
- graphWasTransformed = true;
439
- appliedTransformNames = resolved.names;
443
+ const sessionTransformNames = resolved.names.filter(isSessionCapabilityTransform);
444
+ if (graphTransforms.length > 0) {
445
+ const composed = composeTransforms(...graphTransforms);
446
+ const transformed = composed(execution, {
447
+ capabilities,
448
+ platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
449
+ activationDtype,
450
+ mathDtype,
451
+ accumDtype,
452
+ kvDtype,
453
+ modelId,
454
+ layerTypes,
455
+ });
456
+ if (transformed !== execution) {
457
+ execution = transformed;
458
+ graphWasTransformed = true;
459
+ appliedTransformNames.push(...resolved.names.filter((name) => !isSessionCapabilityTransform(name)));
460
+ }
461
+ }
462
+ if (sessionTransformNames.length > 0) {
463
+ appliedTransformNames.push(...sessionTransformNames);
464
+ }
465
+ if (appliedTransformNames.length > 0) {
440
466
  log.info(
441
467
  'ExecutionV1',
442
- `Capability transforms applied: [${resolved.names.join(', ')}] (${resolved.reason})`
468
+ `Capability transforms applied: [${appliedTransformNames.join(', ')}] (${resolved.reason})`
443
469
  );
444
470
  }
445
471
  } else {
@@ -456,6 +482,8 @@ export function compileExecutionV1(options = {}) {
456
482
  capabilities,
457
483
  platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
458
484
  activationDtype,
485
+ mathDtype,
486
+ accumDtype,
459
487
  kvDtype,
460
488
  headDim,
461
489
  modelId,
@@ -486,25 +514,33 @@ export function compileExecutionV1(options = {}) {
486
514
  // KV cache and all compute dtypes must also be f32.
487
515
  const activationWidened = appliedTransformNames.includes('widenToF32Activations');
488
516
  const fullF32 = activationWidened && capabilities?.hasF16 === false;
489
- const useQwenSelectiveF16Primary =
490
- appliedTransformNames.includes('useQwenF16PrimaryMatmuls')
517
+ const useSelectiveF16Primary =
518
+ (
519
+ appliedTransformNames.includes('useQwenF16PrimaryMatmuls')
520
+ || appliedTransformNames.includes('useGemma4Int4PleSelectiveF16Decode')
521
+ )
491
522
  && !appliedTransformNames.includes('narrowToF16Activations');
523
+ const retainQ4KMaterializationDisabled =
524
+ appliedTransformNames.includes('disableRetainQ4KMaterialization');
492
525
  let effectiveSession = session;
493
- if (activationWidened || useQwenSelectiveF16Primary) {
526
+ if (activationWidened || useSelectiveF16Primary || retainQ4KMaterializationDisabled) {
494
527
  const f32Defaults = fullF32
495
528
  ? { activationDtype: 'f32', mathDtype: 'f32', accumDtype: 'f32', outputDtype: 'f32' }
496
- : useQwenSelectiveF16Primary
529
+ : useSelectiveF16Primary
497
530
  ? { activationDtype: 'f32', outputDtype: 'f32' }
498
531
  : { activationDtype: 'f32' };
499
532
  effectiveSession = {
500
533
  ...session,
501
- compute: {
502
- ...session.compute,
503
- defaults: {
504
- ...session.compute.defaults,
505
- ...f32Defaults,
534
+ ...(retainQ4KMaterializationDisabled ? { retainQ4KMaterialization: false } : {}),
535
+ ...(activationWidened || useSelectiveF16Primary ? {
536
+ compute: {
537
+ ...session.compute,
538
+ defaults: {
539
+ ...session.compute.defaults,
540
+ ...f32Defaults,
541
+ },
506
542
  },
507
- },
543
+ } : {}),
508
544
  ...(fullF32 && session?.kvcache ? {
509
545
  kvcache: {
510
546
  ...session.kvcache,
@@ -570,6 +606,37 @@ export function compileExecutionV1(options = {}) {
570
606
  }
571
607
  }
572
608
 
609
+ // Lane integrity: capture the pre-transform (manifest+profile-agreed) lane
610
+ // and the post-transform (actually dispatched) lane so receipts can honestly
611
+ // surface a capability-driven widening. The manifest-binding gate ensures
612
+ // the pre-transform session already matches manifest quantizationInfo.compute;
613
+ // any drift here is owned by capability transforms (e.g., widenToF32Activations
614
+ // on hasF16=false GPUs).
615
+ const declaredLane = {
616
+ activationDtype,
617
+ mathDtype,
618
+ accumDtype,
619
+ kvDtype,
620
+ };
621
+ const executedLane = {
622
+ activationDtype: effectiveSession.compute?.defaults?.activationDtype ?? null,
623
+ mathDtype: effectiveSession.compute?.defaults?.mathDtype ?? null,
624
+ accumDtype: effectiveSession.compute?.defaults?.accumDtype ?? null,
625
+ kvDtype: effectiveSession.kvcache?.kvDtype ?? effectiveSession.compute?.defaults?.activationDtype ?? null,
626
+ };
627
+ const laneFieldDelta = (
628
+ declaredLane.activationDtype !== executedLane.activationDtype
629
+ || declaredLane.mathDtype !== executedLane.mathDtype
630
+ || declaredLane.accumDtype !== executedLane.accumDtype
631
+ || declaredLane.kvDtype !== executedLane.kvDtype
632
+ );
633
+ const laneIntegrity = {
634
+ declared: declaredLane,
635
+ executed: executedLane,
636
+ status: laneFieldDelta ? 'transformed' : 'matches',
637
+ transforms: [...appliedTransformNames],
638
+ };
639
+
573
640
  const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
574
641
  logIncompatibleOps: !(kernelPath && inlineKernelPathEnabled),
575
642
  ffnDtypeFallback: requireSessionActivationDtype(effectiveSession),
@@ -600,6 +667,7 @@ export function compileExecutionV1(options = {}) {
600
667
  ...(layerPipeline ? { pipeline: layerPipeline } : {}),
601
668
  },
602
669
  appliedTransforms: appliedTransformNames,
670
+ laneIntegrity,
603
671
  fallbackKernelPath,
604
672
  };
605
673
  }
@@ -621,6 +689,7 @@ export function compileExecutionV1(options = {}) {
621
689
  export function applyExecutionV1RuntimeConfig(options = {}) {
622
690
  const runtimeConfig = options.runtimeConfig ?? null;
623
691
  const runtimeOverrides = options.runtimeOverrides;
692
+ const hasExplicitRuntimeOverrides = runtimeOverrides != null;
624
693
  const manifest = options.manifest ?? null;
625
694
  if (!runtimeConfig || !manifest?.inference) {
626
695
  return { runtimeConfig, executionV1State: null };
@@ -644,10 +713,10 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
644
713
  headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
645
714
  capabilities: options.capabilities ?? null,
646
715
  platform: options.platform ?? null,
647
- runtimeSession: hasOwnProperty(options, 'runtimeOverrides')
716
+ runtimeSession: hasExplicitRuntimeOverrides
648
717
  ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'session')
649
718
  : (runtimeConfig.inference?.session ?? null),
650
- runtimeCompute: hasOwnProperty(options, 'runtimeOverrides')
719
+ runtimeCompute: hasExplicitRuntimeOverrides
651
720
  ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'compute')
652
721
  : (runtimeConfig.inference?.compute ?? null),
653
722
  kernelPathPolicy: runtimeConfig.inference?.kernelPathPolicy ?? null,
@@ -12,6 +12,7 @@ import {
12
12
  resolveWeightBufferMaterialization,
13
13
  } from '../../../../gpu/weight-buffer.js';
14
14
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
15
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
15
16
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
16
17
  import {
17
18
  runFusedFFN,
@@ -27,6 +28,7 @@ import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
27
28
  import { applyLoRA } from '../lora-apply.js';
28
29
  import { getLoRAModule } from '../lora.js';
29
30
  import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
31
+ import { runProbes } from '../probes.js';
30
32
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
31
33
  import {
32
34
  getKernelPathMatmulPrecision,
@@ -145,8 +147,20 @@ export function resolveDenseFFNFusedPathDtypes(options = {}) {
145
147
  }
146
148
 
147
149
  function hasExplicitMatmulPrecision(role, phase, layerIdx, kernelPath) {
150
+ // Only treat precision as "explicit split" when declared on the role's OWN
151
+ // step. The FUSED_FFN_PRECISION_FALLBACK_ROLES fallback that resolves via
152
+ // the aggregate `ffn` step's precision applies identically to gate/up/down
153
+ // and must NOT force the split path — otherwise a manifest-declared `ffn`
154
+ // precision permanently blocks the fused gate_up_activation kernel.
148
155
  const precision = getKernelPathMatmulPrecision(role, phase, layerIdx, kernelPath);
149
- return precision?.inputDtype != null || precision?.outputDtype != null;
156
+ if (!precision) return false;
157
+ const fallbackPrecision = getKernelPathMatmulPrecision('ffn', phase, layerIdx, kernelPath);
158
+ const inheritedFromFfn = fallbackPrecision
159
+ && fallbackPrecision.inputDtype === precision.inputDtype
160
+ && fallbackPrecision.outputDtype === precision.outputDtype
161
+ && fallbackPrecision.activationDtype === precision.activationDtype;
162
+ if (inheritedFromFfn) return false;
163
+ return precision.inputDtype != null || precision.outputDtype != null;
150
164
  }
151
165
 
152
166
  export function canUseNativeF16FusedGateUp(options = {}) {
@@ -325,7 +339,7 @@ export async function runDenseFFNGPU(
325
339
  if (!device) throw new Error('No GPU device');
326
340
 
327
341
  const { config, recorder } = context;
328
- const { hiddenSize, hiddenActivation, swigluLimit } = config;
342
+ const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
329
343
  const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
330
344
  const lastTokenIdx = Math.max(0, numTokens - 1);
331
345
  const lora = context.lora || null;
@@ -434,6 +448,22 @@ export async function runDenseFFNGPU(
434
448
  releaseBuffer(gateUpOutput.buffer);
435
449
  }
436
450
 
451
+ // Opt-in WideTile+residual fusion: if the caller (processFFNStandard)
452
+ // staged a residual tensor on the context AND no LoRA is present on
453
+ // down_proj (LoRA requires the pre-residual output for its add), route
454
+ // this matmul to the q4_fused_widetile_residual variant which produces
455
+ // (ffn_down_result + residual) in one dispatch. Tell the caller via a
456
+ // context flag so processFFNStandard skips its downstream doResidualAdd.
457
+ const pendingResidual = context.__pendingFfnResidualTensor;
458
+ const downLoraProbe = getLoRAModule(lora, layerIdx, 'down_proj');
459
+ const mergedSession = getRuntimeConfig().inference?.session;
460
+ const tryFuseDownResidual = pendingResidual != null
461
+ && !downLoraProbe
462
+ && numTokens > 1
463
+ && activatedOutput.dtype === 'f32'
464
+ && downOutputDtype === 'f32'
465
+ && pendingResidual.dtype === 'f32';
466
+ let residualFusedHere = false;
437
467
  let output = await doMatmul(
438
468
  activatedOutput, downWeight,
439
469
  numTokens, hiddenSize, intermediateSize,
@@ -445,9 +475,26 @@ export async function runDenseFFNGPU(
445
475
  outputDtype: downOutputDtype,
446
476
  role: 'ffn_down',
447
477
  executionPolicies: context.executionPolicies ?? null,
478
+ residualTensor: tryFuseDownResidual ? pendingResidual : null,
448
479
  },
449
480
  recorder
450
481
  );
482
+ // Detect whether the fusion fired: if the selected variant for ffn_down
483
+ // matmul was q4_fused_widetile_residual, then output IS post-residual.
484
+ // We infer this cheaply by re-checking the conditions the selector uses.
485
+ // (A cleaner signal would require a return-shape change across all
486
+ // dense.js paths; this local signal is enough for correctness.)
487
+ {
488
+ if (tryFuseDownResidual
489
+ && getKernelCapabilities().hasF16 === true
490
+ && mergedSession?.useWideTileResidualFusion === true
491
+ && mergedSession?.useWideTileQ4KPrefill === true
492
+ && mergedSession?.retainQ4KMaterialization === true
493
+ ) {
494
+ residualFusedHere = true;
495
+ context.__ffnResidualFusedFired = true;
496
+ }
497
+ }
451
498
 
452
499
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
453
500
  if (loraDown) {
@@ -537,6 +584,7 @@ export async function runDenseFFNGPU(
537
584
  f16BatchSupported,
538
585
  batchSize: numTokens,
539
586
  hiddenSizeAligned32,
587
+ useDoubleWideMlp: Boolean(useDoubleWideMlp),
540
588
  });
541
589
  const useFusedGateUp = gateUpPathMode === 'split'
542
590
  ? false
@@ -608,6 +656,16 @@ export async function runDenseFFNGPU(
608
656
  releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
609
657
  }
610
658
 
659
+ // Opt-in WideTile+residual fusion (fused-gate-up path).
660
+ const pendingResidualFused = context.__pendingFfnResidualTensor;
661
+ const downLoraProbeFused = getLoRAModule(lora, layerIdx, 'down_proj');
662
+ const mergedSessionFused = getRuntimeConfig().inference?.session;
663
+ const tryFuseDownResidualFused = pendingResidualFused != null
664
+ && !downLoraProbeFused
665
+ && numTokens > 1
666
+ && downInput.dtype === 'f32'
667
+ && fusedDownOutputDtype === 'f32'
668
+ && pendingResidualFused.dtype === 'f32';
611
669
  let output = await doMatmul(
612
670
  downInput,
613
671
  downWeight,
@@ -622,9 +680,20 @@ export async function runDenseFFNGPU(
622
680
  outputDtype: fusedDownOutputDtype,
623
681
  role: 'ffn_down',
624
682
  executionPolicies: context.executionPolicies ?? null,
683
+ residualTensor: tryFuseDownResidualFused ? pendingResidualFused : null,
625
684
  },
626
685
  recorder
627
686
  );
687
+ {
688
+ if (tryFuseDownResidualFused
689
+ && getKernelCapabilities().hasF16 === true
690
+ && mergedSessionFused?.useWideTileResidualFusion === true
691
+ && mergedSessionFused?.useWideTileQ4KPrefill === true
692
+ && mergedSessionFused?.retainQ4KMaterialization === true
693
+ ) {
694
+ context.__ffnResidualFusedFired = true;
695
+ }
696
+ }
628
697
 
629
698
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
630
699
  if (loraDown) {
@@ -748,6 +817,84 @@ export async function runDenseFFNGPU(
748
817
  });
749
818
  sharedInputOwned = sharedInputTensor !== inputTensor;
750
819
  }
820
+ // Opt-in fused gate + up + GeGLU path. Replaces 3 separate dispatches
821
+ // (gate_proj + up_proj + gelu activation) with a single fused kernel when
822
+ // preconditions match: prefill (M>1), f16-materialisable weights + f16
823
+ // activations, gelu activation, no LoRA on gate/up. Gated by
824
+ // runtime.inference.session.useFusedGateUpGelu (default false).
825
+ const gateF16 = resolveWeightBufferMaterialization(layerWeights.gate, 'f16');
826
+ const upF16 = resolveWeightBufferMaterialization(layerWeights.up, 'f16');
827
+ const gateF16Dtype = getWeightDtype(gateF16);
828
+ const upF16Dtype = getWeightDtype(upF16);
829
+ const earlyLoraGate = getLoRAModule(lora, layerIdx, 'gate_proj');
830
+ const earlyLoraUp = getLoRAModule(lora, layerIdx, 'up_proj');
831
+ const fusedGateUpGeluCandidate = context.useFusedGateUpGelu === true
832
+ && numTokens > 1
833
+ && hiddenActivation === 'gelu'
834
+ && !earlyLoraGate && !earlyLoraUp
835
+ && sharedInputTensor.dtype === 'f16'
836
+ && gateF16Dtype === 'f16'
837
+ && upF16Dtype === 'f16';
838
+ if (fusedGateUpGeluCandidate) {
839
+ const { runFusedGateUpGelu, recordFusedGateUpGelu } =
840
+ await import('../../../../gpu/kernels/fused-gate-up-gelu.js');
841
+ const fused = recorder
842
+ ? await recordFusedGateUpGelu(recorder, sharedInputTensor, gateF16, upF16, {
843
+ M: numTokens,
844
+ hiddenSize,
845
+ intermediateSize,
846
+ transposeB: true,
847
+ })
848
+ : await runFusedGateUpGelu(sharedInputTensor, gateF16, upF16, {
849
+ M: numTokens,
850
+ hiddenSize,
851
+ intermediateSize,
852
+ transposeB: true,
853
+ });
854
+ if (sharedInputOwned) {
855
+ if (recorder) { recorder.trackTemporaryBuffer(sharedInputTensor.buffer); }
856
+ else { releaseBuffer(sharedInputTensor.buffer); }
857
+ }
858
+ // Proceed directly to down_proj with the fused activation output.
859
+ const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
860
+ let downInputTensor = fused;
861
+ let downInputOwned = false;
862
+ if (downInputDtype && fused.dtype !== downInputDtype) {
863
+ downInputTensor = await coerceTensorDtype(fused, downInputDtype, recorder, {
864
+ executionPolicies: context.executionPolicies ?? null,
865
+ op: 'ffn_down_input',
866
+ transitionDeclaredBy: 'step_precision',
867
+ });
868
+ downInputOwned = downInputTensor !== fused;
869
+ }
870
+ let outFused = await doMatmul(
871
+ downInputTensor,
872
+ downWeight,
873
+ numTokens,
874
+ hiddenSize,
875
+ intermediateSize,
876
+ {
877
+ transposeB: 'auto',
878
+ label: `L${layerIdx}.ffn_down`,
879
+ layerIdx,
880
+ kernelPath,
881
+ outputDtype: downOutputDtype,
882
+ role: 'ffn_down',
883
+ executionPolicies: context.executionPolicies ?? null,
884
+ },
885
+ recorder
886
+ );
887
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
888
+ releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
889
+ }
890
+ if (downInputOwned) {
891
+ if (recorder) { recorder.trackTemporaryBuffer(downInputTensor.buffer); }
892
+ else { releaseBuffer(downInputTensor.buffer); }
893
+ }
894
+ if (recorder) { recorder.trackTemporaryBuffer(fused.buffer); }
895
+ else { releaseBuffer(fused.buffer); }
896
+ return outFused;
897
+ }
751
898
  const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
752
899
  let gateOutput = await doMatmul(
753
900
  gateInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
@@ -849,6 +996,25 @@ export async function runDenseFFNGPU(
849
996
  });
850
997
  }
851
998
 
999
+ await runProbes('ffn_gate', gateOutput.buffer, {
1000
+ layerIdx,
1001
+ numTokens,
1002
+ hiddenSize: intermediateSize,
1003
+ probes: context.debugProbes,
1004
+ recorder,
1005
+ operatorDiagnostics: context.operatorDiagnostics,
1006
+ dtype: gateOutput.dtype,
1007
+ });
1008
+ await runProbes('ffn_up', upOutput.buffer, {
1009
+ layerIdx,
1010
+ numTokens,
1011
+ hiddenSize: intermediateSize,
1012
+ probes: context.debugProbes,
1013
+ recorder,
1014
+ operatorDiagnostics: context.operatorDiagnostics,
1015
+ dtype: upOutput.dtype,
1016
+ });
1017
+
852
1018
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
853
1019
  size: numTokens * intermediateSize,
854
1020
  gate: gateOutput,
@@ -866,6 +1032,16 @@ export async function runDenseFFNGPU(
866
1032
  });
867
1033
  }
868
1034
 
1035
+ await runProbes('ffn_act', activatedOutput.buffer, {
1036
+ layerIdx,
1037
+ numTokens,
1038
+ hiddenSize: intermediateSize,
1039
+ probes: context.debugProbes,
1040
+ recorder,
1041
+ operatorDiagnostics: context.operatorDiagnostics,
1042
+ dtype: activatedOutput.dtype,
1043
+ });
1044
+
869
1045
  if (recorder) {
870
1046
  recorder.trackTemporaryBuffer(gateOutput.buffer);
871
1047
  recorder.trackTemporaryBuffer(upOutput.buffer);
@@ -975,7 +1151,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
975
1151
  if (!device) throw new Error('No GPU device');
976
1152
 
977
1153
  const { config, weightConfig, debugFlags, recorder } = context;
978
- const { hiddenSize, hiddenActivation, swigluLimit } = config;
1154
+ const { hiddenSize, hiddenActivation, swigluLimit, useDoubleWideMlp } = config;
979
1155
  const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
980
1156
  const lora = context.lora || null;
981
1157
  const ffnStepPrecision = context.ffnStepPrecision ?? null;
@@ -1124,6 +1300,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
1124
1300
  f16BatchSupported: getKernelCapabilities().hasF16,
1125
1301
  batchSize: numTokens,
1126
1302
  hiddenSizeAligned32,
1303
+ useDoubleWideMlp: Boolean(useDoubleWideMlp),
1127
1304
  });
1128
1305
  const canUseFusedGateUp = gateUpPathMode === 'split'
1129
1306
  ? false
@@ -77,6 +77,18 @@ export async function processFFNStandard(
77
77
  await debugFFNBuffer(context, layerIdx, 'FFN input', normedTensor, numTokens, hiddenSize);
78
78
 
79
79
  // 2. FFN
80
+ // Stage the residual tensor so that runDenseFFNGPU's ffn_down matmul can
81
+ // opportunistically fuse with it (WideTile+residual kernel). The flag
82
+ // `context.__ffnResidualFusedFired` will be set to true by dense.js if
83
+ // the fusion actually fired; we read it below to skip the separate add.
84
+ // If the pre-norm residual sum was captured (fused RMSNorm path), use that
85
+ // buffer directly — it IS the correct residual for ffn_residual because
86
+ // ffn_residual adds to the pre-rmsnorm tensor.
87
+ const pendingResidualForFfn = prenormSumBuffer
88
+ ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
89
+ : postAttn;
90
+ context.__pendingFfnResidualTensor = pendingResidualForFfn;
91
+ context.__ffnResidualFusedFired = false;
80
92
 
81
93
  let ffnOutput;
82
94
  if (config.useMoE && isMoELayerLocal(layerIdx, config, layerWeights)) {
@@ -84,6 +96,9 @@ export async function processFFNStandard(
84
96
  } else {
85
97
  ffnOutput = await runDenseFFNGPU(layerIdx, normedTensor, numTokens, context, layerWeights);
86
98
  }
99
+ const ffnResidualFused = context.__ffnResidualFusedFired === true;
100
+ context.__pendingFfnResidualTensor = null;
101
+ context.__ffnResidualFusedFired = false;
87
102
  await runProbes('ffn_out', ffnOutput.buffer, {
88
103
  layerIdx,
89
104
  numTokens,
@@ -96,20 +111,28 @@ export async function processFFNStandard(
96
111
  await debugFFNBuffer(context, layerIdx, 'FFN output', ffnOutput, numTokens, hiddenSize);
97
112
 
98
113
  // 3. Residual add (uses prenorm sum when fused, otherwise postAttn)
99
- const residualTensor = prenormSumBuffer
100
- ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
101
- : postAttn;
114
+ // When WideTile+residual fusion fired inside runDenseFFNGPU, ffnOutput
115
+ // IS already post-residual — skip the separate add entirely.
116
+ let output;
102
117
  let residualInput = ffnOutput;
103
- const residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
104
- if (residualInputOwned) {
105
- residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
118
+ let residualInputOwned = false;
119
+ if (ffnResidualFused) {
120
+ output = ffnOutput;
121
+ } else {
122
+ const residualTensor = prenormSumBuffer
123
+ ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
124
+ : postAttn;
125
+ residualInputOwned = ffnOutput.dtype !== residualTensor.dtype;
126
+ if (residualInputOwned) {
127
+ residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
128
+ }
129
+ output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
130
+ label: `L${layerIdx}.ffn_residual`,
131
+ layerIdx,
132
+ outputBuffer: decodeOutputBuffer,
133
+ executionPolicies: context.executionPolicies ?? null,
134
+ });
106
135
  }
107
- const output = await doResidualAdd(residualInput, residualTensor, size, recorder, {
108
- label: `L${layerIdx}.ffn_residual`,
109
- layerIdx,
110
- outputBuffer: decodeOutputBuffer,
111
- executionPolicies: context.executionPolicies ?? null,
112
- });
113
136
  await runProbes('layer_out', output.buffer, {
114
137
  layerIdx,
115
138
  numTokens,
@@ -131,7 +154,11 @@ export async function processFFNStandard(
131
154
  if (residualInputOwned) {
132
155
  releaseOrTrack(recorder, residualInput.buffer, decodeBuffers);
133
156
  }
134
- releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
157
+ // When fusion fired, `output === ffnOutput` and we must NOT release it
158
+ // here — it's our return value.
159
+ if (!ffnResidualFused) {
160
+ releaseOrTrack(recorder, ffnOutput.buffer, decodeBuffers);
161
+ }
135
162
 
136
163
  return output;
137
164
  }