doppler-gpu 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +21 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -5,7 +5,12 @@ import {
5
5
  releaseOrTrack
6
6
  } from '../ops.js';
7
7
  import { createTensor } from '../../../../gpu/tensor.js';
8
- import { isWeightBuffer } from '../../../../gpu/weight-buffer.js';
8
+ import {
9
+ getWeightDtype,
10
+ isGpuBufferInstance,
11
+ isWeightBuffer,
12
+ resolveWeightBufferMaterialization,
13
+ } from '../../../../gpu/weight-buffer.js';
9
14
  import { getDevice, getKernelCapabilities } from '../../../../gpu/device.js';
10
15
  import { acquireBuffer, releaseBuffer } from '../../../../memory/buffer-pool.js';
11
16
  import {
@@ -17,12 +22,18 @@ import {
17
22
  recordCastF32ToF16,
18
23
  isFusedQ4KDisabled
19
24
  } from '../../../../gpu/kernel-selector.js';
20
- import { log } from '../../../../debug/index.js';
25
+ import { log, trace } from '../../../../debug/index.js';
21
26
  import { isKernelDebugEnabled, dumpTokenVector } from '../debug-utils.js';
22
27
  import { applyLoRA } from '../lora-apply.js';
23
28
  import { getLoRAModule } from '../lora.js';
24
29
  import { getWeightBuffer, getNormWeightBuffer } from '../weights.js';
25
30
  import { selectRuleValue } from '../../../../rules/rule-registry.js';
31
+ import {
32
+ getKernelPathMatmulPrecision,
33
+ getKernelPathMatmulVariant,
34
+ } from '../../../../config/kernel-path-loader.js';
35
+ import { resolveLayerIntermediateSize } from '../config.js';
36
+ import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
26
37
 
27
38
  const ACTIVATION_FN_MAP = {
28
39
  gelu: doGeLU,
@@ -33,6 +44,210 @@ function resolveActivationOp(hiddenActivation) {
33
44
  return selectRuleValue('inference', 'ffn', 'activationOp', { hiddenActivation });
34
45
  }
35
46
 
47
+ function hasQ4KMaterialization(weight) {
48
+ return isWeightBuffer(weight) && !!weight.materializations?.q4k?.buffer;
49
+ }
50
+
51
+ function isQ4KMatmulVariant(variant) {
52
+ return typeof variant === 'string' && variant.startsWith('q4_');
53
+ }
54
+
55
+ export function resolveDenseFFNMatmulStepDtype(options = {}) {
56
+ const precision = getKernelPathMatmulPrecision(
57
+ options.role,
58
+ options.phase,
59
+ options.layerIdx,
60
+ options.kernelPath
61
+ );
62
+ const requested = precision?.[options.field]
63
+ ?? options.ffnStepPrecision?.[options.field]
64
+ ?? options.fallback;
65
+ if (requested == null) {
66
+ return options.fallback;
67
+ }
68
+ return selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype: requested });
69
+ }
70
+
71
+ function resolveMatmulStepDtype(role, phase, layerIdx, kernelPath, fallback, field, ffnStepPrecision = null) {
72
+ return resolveDenseFFNMatmulStepDtype({
73
+ role,
74
+ phase,
75
+ layerIdx,
76
+ kernelPath,
77
+ fallback,
78
+ field,
79
+ ffnStepPrecision,
80
+ });
81
+ }
82
+
83
+ export function resolveDenseFFNFusedPathDtypes(options = {}) {
84
+ const phase = options.phase ?? null;
85
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
86
+ const kernelPath = options.kernelPath ?? null;
87
+ const ffnStepPrecision = options.ffnStepPrecision ?? null;
88
+ const fallbackInputDtype = options.fallbackInputDtype ?? null;
89
+ const fallbackOutputDtype = options.fallbackOutputDtype ?? fallbackInputDtype;
90
+
91
+ const fusedGateUpInputDtype = resolveDenseFFNMatmulStepDtype({
92
+ role: 'ffn_gate_up',
93
+ phase,
94
+ layerIdx,
95
+ kernelPath,
96
+ fallback: null,
97
+ field: 'inputDtype',
98
+ ffnStepPrecision,
99
+ });
100
+ const gateInputDtype = resolveDenseFFNMatmulStepDtype({
101
+ role: 'ffn_gate',
102
+ phase,
103
+ layerIdx,
104
+ kernelPath,
105
+ fallback: null,
106
+ field: 'inputDtype',
107
+ ffnStepPrecision,
108
+ });
109
+ const upInputDtype = resolveDenseFFNMatmulStepDtype({
110
+ role: 'ffn_up',
111
+ phase,
112
+ layerIdx,
113
+ kernelPath,
114
+ fallback: null,
115
+ field: 'inputDtype',
116
+ ffnStepPrecision,
117
+ });
118
+ const resolvedFusedGateUpInputDtype = fusedGateUpInputDtype
119
+ ?? (gateInputDtype && gateInputDtype === upInputDtype ? gateInputDtype : fallbackInputDtype);
120
+
121
+ const fusedGateUpOutputDtype = resolveDenseFFNMatmulStepDtype({
122
+ role: 'ffn_gate_up',
123
+ phase,
124
+ layerIdx,
125
+ kernelPath,
126
+ fallback: fallbackOutputDtype,
127
+ field: 'outputDtype',
128
+ ffnStepPrecision,
129
+ });
130
+ const downInputDtype = resolveDenseFFNMatmulStepDtype({
131
+ role: 'ffn_down',
132
+ phase,
133
+ layerIdx,
134
+ kernelPath,
135
+ fallback: fusedGateUpOutputDtype,
136
+ field: 'inputDtype',
137
+ ffnStepPrecision,
138
+ });
139
+
140
+ return {
141
+ fusedGateUpInputDtype: resolvedFusedGateUpInputDtype,
142
+ fusedGateUpOutputDtype,
143
+ downInputDtype,
144
+ };
145
+ }
146
+
147
+ function hasExplicitMatmulPrecision(role, phase, layerIdx, kernelPath) {
148
+ const precision = getKernelPathMatmulPrecision(role, phase, layerIdx, kernelPath);
149
+ return precision?.inputDtype != null || precision?.outputDtype != null;
150
+ }
151
+
152
+ export function canUseNativeF16FusedGateUp(options = {}) {
153
+ if (options.inputDtype !== 'f16' || options.hasF16 !== true) {
154
+ return false;
155
+ }
156
+ return options.gateDtype === 'f16' || options.gateDtype === 'q4k';
157
+ }
158
+
159
+ async function coerceTensorDtype(tensor, targetDtype, recorder, options = {}) {
160
+ if (!targetDtype || tensor.dtype === targetDtype) {
161
+ return tensor;
162
+ }
163
+ assertImplicitDtypeTransitionAllowed({
164
+ executionPolicies: options.executionPolicies ?? null,
165
+ fromDtype: tensor.dtype,
166
+ toDtype: targetDtype,
167
+ op: options.op ?? 'ffn',
168
+ detail: 'The execution graph must declare this cast explicitly.',
169
+ transitionDeclaredBy: options.transitionDeclaredBy ?? null,
170
+ });
171
+ if (tensor.dtype === 'f32' && targetDtype === 'f16') {
172
+ return recorder ? await recordCastF32ToF16(recorder, tensor) : await castF32ToF16(tensor);
173
+ }
174
+ if (tensor.dtype === 'f16' && targetDtype === 'f32') {
175
+ return recorder ? await recordCastF16ToF32(recorder, tensor) : await castF16ToF32(tensor);
176
+ }
177
+ throw new Error(`Unsupported FFN matmul dtype coercion: ${tensor.dtype} -> ${targetDtype}`);
178
+ }
179
+
180
+ export function resolveGateUpPathMode(options = {}) {
181
+ const kernelPath = options.kernelPath ?? null;
182
+ const phase = options.phase ?? null;
183
+ const layerIdx = Number.isFinite(options.layerIdx) ? options.layerIdx : 0;
184
+ if (!kernelPath || !phase) {
185
+ return 'implicit';
186
+ }
187
+
188
+ const fusedVariant = getKernelPathMatmulVariant('ffn_gate_up', phase, layerIdx, kernelPath);
189
+ if (fusedVariant != null) {
190
+ return 'fused';
191
+ }
192
+
193
+ const gateVariant = getKernelPathMatmulVariant('ffn_gate', phase, layerIdx, kernelPath);
194
+ const upVariant = getKernelPathMatmulVariant('ffn_up', phase, layerIdx, kernelPath);
195
+ const hasExplicitGatePrecision = hasExplicitMatmulPrecision('ffn_gate', phase, layerIdx, kernelPath);
196
+ const hasExplicitUpPrecision = hasExplicitMatmulPrecision('ffn_up', phase, layerIdx, kernelPath);
197
+ const hasExplicitDownPrecision = hasExplicitMatmulPrecision('ffn_down', phase, layerIdx, kernelPath);
198
+ const hasExplicitSplitPrecision = hasExplicitGatePrecision || hasExplicitUpPrecision || hasExplicitDownPrecision;
199
+ if (hasExplicitSplitPrecision) {
200
+ const decodeQ4PrecisionCanStayFused = phase === 'decode'
201
+ && !hasExplicitDownPrecision
202
+ && isQ4KMatmulVariant(gateVariant)
203
+ && gateVariant === upVariant
204
+ && (hasExplicitGatePrecision || hasExplicitUpPrecision);
205
+ if (!decodeQ4PrecisionCanStayFused) {
206
+ return 'split';
207
+ }
208
+ }
209
+ if (
210
+ gateVariant != null
211
+ && upVariant != null
212
+ ) {
213
+ if (
214
+ phase === 'prefill'
215
+ && !isQ4KMatmulVariant(gateVariant)
216
+ && !isQ4KMatmulVariant(upVariant)
217
+ ) {
218
+ return 'split';
219
+ }
220
+ }
221
+
222
+ return 'implicit';
223
+ }
224
+
225
+ export function resolveFusedGateUpWeights(layerWeights, options = {}) {
226
+ const gate = layerWeights?.gate ?? null;
227
+ const up = layerWeights?.up ?? null;
228
+ const hiddenSize = Number.isFinite(options.hiddenSize) ? options.hiddenSize : 0;
229
+ const q4kAllowed = !isFusedQ4KDisabled({ kernelPath: options.kernelPath ?? null });
230
+ const hasMixedQ4KMaterialization = hasQ4KMaterialization(gate) && hasQ4KMaterialization(up);
231
+ const preferQ4KMaterialization = hiddenSize > 0
232
+ && hiddenSize % 32 === 0
233
+ && q4kAllowed
234
+ && hasMixedQ4KMaterialization;
235
+ const resolvedGate = preferQ4KMaterialization
236
+ ? resolveWeightBufferMaterialization(gate, 'q4k')
237
+ : gate;
238
+ const resolvedUp = preferQ4KMaterialization
239
+ ? resolveWeightBufferMaterialization(up, 'q4k')
240
+ : up;
241
+
242
+ return {
243
+ gate: resolvedGate,
244
+ up: resolvedUp,
245
+ gateDtype: resolvedGate ? getWeightDtype(resolvedGate) : null,
246
+ upDtype: resolvedUp ? getWeightDtype(resolvedUp) : null,
247
+ hasQ4KMaterialization: hasMixedQ4KMaterialization,
248
+ };
249
+ }
250
+
36
251
  async function dispatchActivation(hiddenActivation, input, options, recorder) {
37
252
  const op = resolveActivationOp(hiddenActivation);
38
253
  const fn = ACTIVATION_FN_MAP[op];
@@ -53,11 +268,26 @@ async function dispatchFusedGateUp({
53
268
  hiddenActivation,
54
269
  swigluLimit,
55
270
  recorder,
271
+ executionPolicies = null,
56
272
  }) {
57
- const useNativeF16Fused = inputTensor.dtype === 'f16' && gateDtype === 'f16' && getKernelCapabilities().hasF16;
58
- const fusedInput = (!useNativeF16Fused && inputTensor.dtype === 'f16')
59
- ? (recorder ? await recordCastF16ToF32(recorder, inputTensor) : await castF16ToF32(inputTensor))
60
- : inputTensor;
273
+ const useNativeF16Fused = canUseNativeF16FusedGateUp({
274
+ inputDtype: inputTensor.dtype,
275
+ gateDtype,
276
+ hasF16: getKernelCapabilities().hasF16,
277
+ });
278
+ let fusedInput = inputTensor;
279
+ if (!useNativeF16Fused && inputTensor.dtype === 'f16') {
280
+ assertImplicitDtypeTransitionAllowed({
281
+ executionPolicies,
282
+ fromDtype: 'f16',
283
+ toDtype: 'f32',
284
+ op: 'ffn_gate_up',
285
+ detail: 'The fused FFN kernel would widen activations internally.',
286
+ });
287
+ fusedInput = recorder
288
+ ? await recordCastF16ToF32(recorder, inputTensor)
289
+ : await castF16ToF32(inputTensor);
290
+ }
61
291
 
62
292
  if (recorder && fusedInput !== inputTensor) {
63
293
  recorder.trackTemporaryBuffer(fusedInput.buffer);
@@ -95,20 +325,42 @@ export async function runDenseFFNGPU(
95
325
  if (!device) throw new Error('No GPU device');
96
326
 
97
327
  const { config, recorder } = context;
98
- const { hiddenSize, intermediateSize, hiddenActivation, swigluLimit } = config;
328
+ const { hiddenSize, hiddenActivation, swigluLimit } = config;
329
+ const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
99
330
  const lastTokenIdx = Math.max(0, numTokens - 1);
100
331
  const lora = context.lora || null;
332
+ const ffnStepPrecision = context.ffnStepPrecision ?? null;
101
333
  const kernelPath = context.kernelPath ?? null;
334
+ const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
335
+ const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
102
336
 
103
337
  if (layerWeights?.gateUp && layerWeights?.down) {
104
338
  const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
105
339
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
106
340
 
107
341
  const useF16 = inputTensor.dtype === 'f16';
108
- const matmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
342
+ const defaultMatmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
109
343
  useF16,
110
344
  fallback: inputTensor.dtype,
111
345
  });
346
+ const matmulOutputDtype = resolveMatmulStepDtype(
347
+ 'ffn_gate_up',
348
+ phase,
349
+ layerIdx,
350
+ kernelPath,
351
+ defaultMatmulOutputDtype,
352
+ 'outputDtype',
353
+ ffnStepPrecision
354
+ );
355
+ const downOutputDtype = resolveMatmulStepDtype(
356
+ 'ffn_down',
357
+ phase,
358
+ layerIdx,
359
+ kernelPath,
360
+ 'f32',
361
+ 'outputDtype',
362
+ ffnStepPrecision
363
+ );
112
364
  let gateUpOutput = await doMatmul(
113
365
  inputTensor, gateUpWeight,
114
366
  numTokens, intermediateSize * 2, hiddenSize,
@@ -119,6 +371,7 @@ export async function runDenseFFNGPU(
119
371
  kernelPath,
120
372
  outputDtype: matmulOutputDtype,
121
373
  role: 'ffn_gate_up',
374
+ executionPolicies: context.executionPolicies ?? null,
122
375
  },
123
376
  recorder
124
377
  );
@@ -153,7 +406,7 @@ export async function runDenseFFNGPU(
153
406
  });
154
407
  }
155
408
 
156
- if (!(layerWeights.gateUp instanceof GPUBuffer) && !isWeightBuffer(layerWeights.gateUp)) {
409
+ if (!isGpuBufferInstance(layerWeights.gateUp) && !isWeightBuffer(layerWeights.gateUp)) {
157
410
  releaseOrTrack(recorder, isWeightBuffer(gateUpWeight) ? gateUpWeight.buffer : gateUpWeight);
158
411
  }
159
412
 
@@ -189,8 +442,9 @@ export async function runDenseFFNGPU(
189
442
  label: `L${layerIdx}.ffn_down`,
190
443
  layerIdx,
191
444
  kernelPath,
192
- outputDtype: matmulOutputDtype,
445
+ outputDtype: downOutputDtype,
193
446
  role: 'ffn_down',
447
+ executionPolicies: context.executionPolicies ?? null,
194
448
  },
195
449
  recorder
196
450
  );
@@ -225,7 +479,7 @@ export async function runDenseFFNGPU(
225
479
  });
226
480
  }
227
481
 
228
- if (!(layerWeights.down instanceof GPUBuffer) && !isWeightBuffer(layerWeights.down)) {
482
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
229
483
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
230
484
  }
231
485
  if (recorder) {
@@ -246,17 +500,29 @@ export async function runDenseFFNGPU(
246
500
  (hasGate ? getLoRAModule(lora, layerIdx, 'gate_proj') : null) ||
247
501
  (hasUp ? getLoRAModule(lora, layerIdx, 'up_proj') : null)
248
502
  );
249
- const gateDtype = hasGate && isWeightBuffer(layerWeights.gate) ? layerWeights.gate.dtype : (hasGate ? 'f32' : null);
250
- const upDtype = hasUp && isWeightBuffer(layerWeights.up) ? layerWeights.up.dtype : (hasUp ? 'f32' : null);
251
- const dtypeMatches = gateDtype != null && upDtype != null && gateDtype === upDtype;
252
- const q4kFusedAllowed = gateDtype !== 'q4k' || !isFusedQ4KDisabled({ kernelPath });
253
- const dtypeSupported = gateDtype === 'f16' || gateDtype === 'f32' || (gateDtype === 'q4k' && q4kFusedAllowed);
503
+ const hiddenSizeAligned32 = hiddenSize % 32 === 0;
254
504
  const activationDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
255
505
  useF16: inputTensor.dtype === 'f16',
256
506
  fallback: inputTensor.dtype,
257
507
  });
508
+ const defaultMatmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
509
+ useF16: inputTensor.dtype === 'f16',
510
+ fallback: inputTensor.dtype,
511
+ });
512
+ const fusedGateUpWeights = resolveFusedGateUpWeights(layerWeights, {
513
+ activationDtype,
514
+ hiddenSize,
515
+ kernelPath,
516
+ phase,
517
+ layerIdx,
518
+ });
519
+ const gateDtype = fusedGateUpWeights.gateDtype ?? (hasGate ? 'f32' : null);
520
+ const upDtype = fusedGateUpWeights.upDtype ?? (hasUp ? 'f32' : null);
521
+ const dtypeMatches = gateDtype != null && upDtype != null && gateDtype === upDtype;
522
+ const q4kFusedAllowed = gateDtype !== 'q4k' || !isFusedQ4KDisabled({ kernelPath });
523
+ const dtypeSupported = gateDtype === 'f16' || gateDtype === 'f32' || (gateDtype === 'q4k' && q4kFusedAllowed);
258
524
  const f16BatchSupported = getKernelCapabilities().hasF16;
259
- const useFusedGateUp = selectRuleValue('inference', 'ffn', 'useFusedGateUp', {
525
+ const useFusedGateUpByRule = selectRuleValue('inference', 'ffn', 'useFusedGateUp', {
260
526
  hasGate,
261
527
  hasUp,
262
528
  hasDown,
@@ -265,40 +531,80 @@ export async function runDenseFFNGPU(
265
531
  hasLoRA,
266
532
  dtypeMatches,
267
533
  dtypeSupported,
534
+ weightDtype: gateDtype,
535
+ hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
268
536
  activationDtype,
269
537
  f16BatchSupported,
270
538
  batchSize: numTokens,
539
+ hiddenSizeAligned32,
271
540
  });
541
+ const useFusedGateUp = gateUpPathMode === 'split'
542
+ ? false
543
+ : useFusedGateUpByRule;
544
+ trace.ffn(
545
+ layerIdx,
546
+ `useFusedGateUp=${useFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
547
+ `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
548
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
549
+ );
272
550
 
273
551
  if (useFusedGateUp) {
274
- const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
275
- const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
276
- const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
277
- const useF16 = inputTensor.dtype === 'f16';
278
- const matmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
279
- useF16,
280
- fallback: inputTensor.dtype,
552
+ const {
553
+ fusedGateUpInputDtype,
554
+ downInputDtype,
555
+ } = resolveDenseFFNFusedPathDtypes({
556
+ phase,
557
+ layerIdx,
558
+ kernelPath,
559
+ ffnStepPrecision,
560
+ fallbackInputDtype: inputTensor.dtype,
561
+ fallbackOutputDtype: defaultMatmulOutputDtype,
281
562
  });
563
+ const gateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
564
+ const upWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
565
+ const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
566
+ const fusedDownOutputDtype = resolveMatmulStepDtype(
567
+ 'ffn_down',
568
+ phase,
569
+ layerIdx,
570
+ kernelPath,
571
+ 'f32',
572
+ 'outputDtype',
573
+ ffnStepPrecision
574
+ );
575
+ let fusedInput = inputTensor;
576
+ let fusedInputOwned = false;
577
+ if (fusedGateUpInputDtype && fusedGateUpInputDtype !== inputTensor.dtype) {
578
+ fusedInput = await coerceTensorDtype(inputTensor, fusedGateUpInputDtype, recorder, {
579
+ executionPolicies: context.executionPolicies ?? null,
580
+ op: 'ffn_gate_up_input',
581
+ transitionDeclaredBy: 'step_precision',
582
+ });
583
+ fusedInputOwned = fusedInput !== inputTensor;
584
+ }
282
585
  const fusedOutput = await dispatchFusedGateUp({
283
- inputTensor, gateWeight, upWeight, gateDtype,
586
+ inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
284
587
  hiddenSize, intermediateSize, numTokens,
285
588
  hiddenActivation, swigluLimit, recorder,
589
+ executionPolicies: context.executionPolicies ?? null,
286
590
  });
287
591
 
288
592
  let downInput = fusedOutput;
289
- if (matmulOutputDtype === 'f16' && fusedOutput.dtype !== 'f16') {
290
- downInput = recorder
291
- ? await recordCastF32ToF16(recorder, fusedOutput)
292
- : await castF32ToF16(fusedOutput);
593
+ if (downInputDtype && fusedOutput.dtype !== downInputDtype) {
594
+ downInput = await coerceTensorDtype(fusedOutput, downInputDtype, recorder, {
595
+ executionPolicies: context.executionPolicies ?? null,
596
+ op: 'ffn_down_input',
597
+ transitionDeclaredBy: 'step_precision',
598
+ });
293
599
  if (recorder) {
294
600
  recorder.trackTemporaryBuffer(downInput.buffer);
295
601
  }
296
602
  }
297
603
 
298
- if (!(layerWeights.gate instanceof GPUBuffer) && !isWeightBuffer(layerWeights.gate)) {
604
+ if (!isGpuBufferInstance(layerWeights.gate) && !isWeightBuffer(layerWeights.gate)) {
299
605
  releaseOrTrack(recorder, isWeightBuffer(gateWeight) ? gateWeight.buffer : gateWeight);
300
606
  }
301
- if (!(layerWeights.up instanceof GPUBuffer) && !isWeightBuffer(layerWeights.up)) {
607
+ if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
302
608
  releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
303
609
  }
304
610
 
@@ -313,8 +619,9 @@ export async function runDenseFFNGPU(
313
619
  label: `L${layerIdx}.ffn_down`,
314
620
  layerIdx,
315
621
  kernelPath,
316
- outputDtype: matmulOutputDtype,
622
+ outputDtype: fusedDownOutputDtype,
317
623
  role: 'ffn_down',
624
+ executionPolicies: context.executionPolicies ?? null,
318
625
  },
319
626
  recorder
320
627
  );
@@ -340,7 +647,7 @@ export async function runDenseFFNGPU(
340
647
  }
341
648
  }
342
649
 
343
- if (!(layerWeights.down instanceof GPUBuffer) && !isWeightBuffer(layerWeights.down)) {
650
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
344
651
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
345
652
  }
346
653
 
@@ -348,11 +655,17 @@ export async function runDenseFFNGPU(
348
655
  if (downInput !== fusedOutput) {
349
656
  recorder.trackTemporaryBuffer(downInput.buffer);
350
657
  }
658
+ if (fusedInputOwned) {
659
+ recorder.trackTemporaryBuffer(fusedInput.buffer);
660
+ }
351
661
  recorder.trackTemporaryBuffer(fusedOutput.buffer);
352
662
  } else {
353
663
  if (downInput !== fusedOutput) {
354
664
  releaseBuffer(downInput.buffer);
355
665
  }
666
+ if (fusedInputOwned) {
667
+ releaseBuffer(fusedInput.buffer);
668
+ }
356
669
  releaseBuffer(fusedOutput.buffer);
357
670
  }
358
671
 
@@ -370,14 +683,74 @@ export async function runDenseFFNGPU(
370
683
  return createTensor(outputBuffer, inputTensor.dtype, [...inputTensor.shape], 'ffn_output_copy');
371
684
  }
372
685
 
373
- const useF16 = inputTensor.dtype === 'f16';
374
- const matmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
375
- useF16,
376
- fallback: inputTensor.dtype,
377
- });
686
+ const gateInputDtype = resolveMatmulStepDtype(
687
+ 'ffn_gate',
688
+ phase,
689
+ layerIdx,
690
+ kernelPath,
691
+ inputTensor.dtype,
692
+ 'inputDtype',
693
+ ffnStepPrecision
694
+ );
695
+ const gateOutputDtype = resolveMatmulStepDtype(
696
+ 'ffn_gate',
697
+ phase,
698
+ layerIdx,
699
+ kernelPath,
700
+ defaultMatmulOutputDtype,
701
+ 'outputDtype',
702
+ ffnStepPrecision
703
+ );
704
+ const upInputDtype = resolveMatmulStepDtype(
705
+ 'ffn_up',
706
+ phase,
707
+ layerIdx,
708
+ kernelPath,
709
+ inputTensor.dtype,
710
+ 'inputDtype',
711
+ ffnStepPrecision
712
+ );
713
+ const upOutputDtype = resolveMatmulStepDtype(
714
+ 'ffn_up',
715
+ phase,
716
+ layerIdx,
717
+ kernelPath,
718
+ defaultMatmulOutputDtype,
719
+ 'outputDtype',
720
+ ffnStepPrecision
721
+ );
722
+ const downOutputDtype = resolveMatmulStepDtype(
723
+ 'ffn_down',
724
+ phase,
725
+ layerIdx,
726
+ kernelPath,
727
+ 'f32',
728
+ 'outputDtype',
729
+ ffnStepPrecision
730
+ );
731
+ const downInputDtype = resolveMatmulStepDtype(
732
+ 'ffn_down',
733
+ phase,
734
+ layerIdx,
735
+ kernelPath,
736
+ downOutputDtype,
737
+ 'inputDtype',
738
+ ffnStepPrecision
739
+ );
740
+ const sharedInputDtype = gateInputDtype === upInputDtype ? gateInputDtype : null;
741
+ let sharedInputTensor = inputTensor;
742
+ let sharedInputOwned = false;
743
+ if (sharedInputDtype && sharedInputDtype !== inputTensor.dtype) {
744
+ sharedInputTensor = await coerceTensorDtype(inputTensor, sharedInputDtype, recorder, {
745
+ executionPolicies: context.executionPolicies ?? null,
746
+ op: 'ffn_shared_input',
747
+ transitionDeclaredBy: 'step_precision',
748
+ });
749
+ sharedInputOwned = sharedInputTensor !== inputTensor;
750
+ }
378
751
  const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
379
752
  let gateOutput = await doMatmul(
380
- inputTensor,
753
+ gateInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
381
754
  gateWeight,
382
755
  numTokens,
383
756
  intermediateSize,
@@ -387,12 +760,13 @@ export async function runDenseFFNGPU(
387
760
  label: `L${layerIdx}.ffn_gate`,
388
761
  layerIdx,
389
762
  kernelPath,
390
- outputDtype: matmulOutputDtype,
763
+ outputDtype: gateOutputDtype,
391
764
  role: 'ffn_gate',
765
+ executionPolicies: context.executionPolicies ?? null,
392
766
  },
393
767
  recorder
394
768
  );
395
- if (!(layerWeights.gate instanceof GPUBuffer) && !isWeightBuffer(layerWeights.gate)) {
769
+ if (!isGpuBufferInstance(layerWeights.gate) && !isWeightBuffer(layerWeights.gate)) {
396
770
  releaseOrTrack(recorder, isWeightBuffer(gateWeight) ? gateWeight.buffer : gateWeight);
397
771
  }
398
772
 
@@ -419,7 +793,7 @@ export async function runDenseFFNGPU(
419
793
 
420
794
  const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
421
795
  let upOutput = await doMatmul(
422
- inputTensor,
796
+ upInputDtype === sharedInputTensor.dtype ? sharedInputTensor : inputTensor,
423
797
  upWeight,
424
798
  numTokens,
425
799
  intermediateSize,
@@ -429,12 +803,13 @@ export async function runDenseFFNGPU(
429
803
  label: `L${layerIdx}.ffn_up`,
430
804
  layerIdx,
431
805
  kernelPath,
432
- outputDtype: matmulOutputDtype,
806
+ outputDtype: upOutputDtype,
433
807
  role: 'ffn_up',
808
+ executionPolicies: context.executionPolicies ?? null,
434
809
  },
435
810
  recorder
436
811
  );
437
- if (!(layerWeights.up instanceof GPUBuffer) && !isWeightBuffer(layerWeights.up)) {
812
+ if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
438
813
  releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
439
814
  }
440
815
 
@@ -498,10 +873,27 @@ export async function runDenseFFNGPU(
498
873
  releaseBuffer(gateOutput.buffer);
499
874
  releaseBuffer(upOutput.buffer);
500
875
  }
876
+ if (sharedInputOwned) {
877
+ if (recorder) {
878
+ recorder.trackTemporaryBuffer(sharedInputTensor.buffer);
879
+ } else {
880
+ releaseBuffer(sharedInputTensor.buffer);
881
+ }
882
+ }
501
883
 
502
884
  const downWeight = getWeightBuffer(layerWeights.down, 'ffn_down');
885
+ let downInputTensor = activatedOutput;
886
+ let downInputOwned = false;
887
+ if (downInputDtype && activatedOutput.dtype !== downInputDtype) {
888
+ downInputTensor = await coerceTensorDtype(activatedOutput, downInputDtype, recorder, {
889
+ executionPolicies: context.executionPolicies ?? null,
890
+ op: 'ffn_down_input',
891
+ transitionDeclaredBy: 'step_precision',
892
+ });
893
+ downInputOwned = downInputTensor !== activatedOutput;
894
+ }
503
895
  let output = await doMatmul(
504
- activatedOutput,
896
+ downInputTensor,
505
897
  downWeight,
506
898
  numTokens,
507
899
  hiddenSize,
@@ -511,8 +903,9 @@ export async function runDenseFFNGPU(
511
903
  label: `L${layerIdx}.ffn_down`,
512
904
  layerIdx,
513
905
  kernelPath,
514
- outputDtype: matmulOutputDtype,
906
+ outputDtype: downOutputDtype,
515
907
  role: 'ffn_down',
908
+ executionPolicies: context.executionPolicies ?? null,
516
909
  },
517
910
  recorder
518
911
  );
@@ -520,7 +913,7 @@ export async function runDenseFFNGPU(
520
913
  const loraDown = getLoRAModule(lora, layerIdx, 'down_proj');
521
914
  if (loraDown) {
522
915
  const combined = await applyLoRA(
523
- activatedOutput,
916
+ downInputTensor,
524
917
  output,
525
918
  loraDown,
526
919
  { M: numTokens, N: hiddenSize, K: intermediateSize },
@@ -547,9 +940,16 @@ export async function runDenseFFNGPU(
547
940
  });
548
941
  }
549
942
 
550
- if (!(layerWeights.down instanceof GPUBuffer) && !isWeightBuffer(layerWeights.down)) {
943
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
551
944
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
552
945
  }
946
+ if (downInputOwned) {
947
+ if (recorder) {
948
+ recorder.trackTemporaryBuffer(downInputTensor.buffer);
949
+ } else {
950
+ releaseBuffer(downInputTensor.buffer);
951
+ }
952
+ }
553
953
  if (recorder) {
554
954
  recorder.trackTemporaryBuffer(activatedOutput.buffer);
555
955
  } else {
@@ -575,9 +975,13 @@ export async function runDenseFFNWithFusedPostNormGPU(
575
975
  if (!device) throw new Error('No GPU device');
576
976
 
577
977
  const { config, weightConfig, debugFlags, recorder } = context;
578
- const { hiddenSize, intermediateSize, hiddenActivation, swigluLimit } = config;
978
+ const { hiddenSize, hiddenActivation, swigluLimit } = config;
979
+ const intermediateSize = resolveLayerIntermediateSize(config, layerIdx);
579
980
  const lora = context.lora || null;
981
+ const ffnStepPrecision = context.ffnStepPrecision ?? null;
580
982
  const kernelPath = context.kernelPath ?? null;
983
+ const phase = context.phase ?? (numTokens === 1 ? 'decode' : 'prefill');
984
+ const gateUpPathMode = resolveGateUpPathMode({ kernelPath, phase, layerIdx });
581
985
 
582
986
  if (!layerWeights.down || !layerWeights.postFeedforwardNorm) {
583
987
  throw new Error('Missing down or norm weights');
@@ -589,15 +993,44 @@ export async function runDenseFFNWithFusedPostNormGPU(
589
993
 
590
994
  let activatedOutput;
591
995
  const useF16 = inputTensor.dtype === 'f16';
592
- const matmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
996
+ const defaultMatmulOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
593
997
  useF16,
594
998
  fallback: inputTensor.dtype,
595
999
  });
1000
+ const matmulOutputDtype = resolveMatmulStepDtype(
1001
+ 'ffn_gate_up',
1002
+ phase,
1003
+ layerIdx,
1004
+ kernelPath,
1005
+ defaultMatmulOutputDtype,
1006
+ 'outputDtype',
1007
+ ffnStepPrecision
1008
+ );
1009
+ const {
1010
+ fusedGateUpInputDtype,
1011
+ } = resolveDenseFFNFusedPathDtypes({
1012
+ phase,
1013
+ layerIdx,
1014
+ kernelPath,
1015
+ ffnStepPrecision,
1016
+ fallbackInputDtype: inputTensor.dtype,
1017
+ fallbackOutputDtype: matmulOutputDtype,
1018
+ });
596
1019
 
597
1020
  if (layerWeights.gateUp) {
598
1021
  const gateUpWeight = getWeightBuffer(layerWeights.gateUp, 'ffn_gate_up');
1022
+ let gateUpInput = inputTensor;
1023
+ let gateUpInputOwned = false;
1024
+ if (fusedGateUpInputDtype && fusedGateUpInputDtype !== inputTensor.dtype) {
1025
+ gateUpInput = await coerceTensorDtype(inputTensor, fusedGateUpInputDtype, recorder, {
1026
+ executionPolicies: context.executionPolicies ?? null,
1027
+ op: 'ffn_gate_up_input',
1028
+ transitionDeclaredBy: 'step_precision',
1029
+ });
1030
+ gateUpInputOwned = gateUpInput !== inputTensor;
1031
+ }
599
1032
  let gateUpOutput = await doMatmul(
600
- inputTensor, gateUpWeight,
1033
+ gateUpInput, gateUpWeight,
601
1034
  numTokens, intermediateSize * 2, hiddenSize,
602
1035
  {
603
1036
  transposeB: 'auto',
@@ -606,6 +1039,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
606
1039
  label: `L${layerIdx}.ffn_gate_up`,
607
1040
  layerIdx,
608
1041
  kernelPath,
1042
+ executionPolicies: context.executionPolicies ?? null,
609
1043
  },
610
1044
  recorder
611
1045
  );
@@ -613,7 +1047,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
613
1047
  const loraGateUp = getLoRAModule(lora, layerIdx, 'gate_up_proj');
614
1048
  if (loraGateUp) {
615
1049
  const combined = await applyLoRA(
616
- inputTensor,
1050
+ gateUpInput,
617
1051
  gateUpOutput,
618
1052
  loraGateUp,
619
1053
  { M: numTokens, N: intermediateSize * 2, K: hiddenSize },
@@ -631,7 +1065,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
631
1065
  }
632
1066
  }
633
1067
 
634
- if (!(layerWeights.gateUp instanceof GPUBuffer) && !isWeightBuffer(layerWeights.gateUp)) {
1068
+ if (!isGpuBufferInstance(layerWeights.gateUp) && !isWeightBuffer(layerWeights.gateUp)) {
635
1069
  releaseOrTrack(recorder, isWeightBuffer(gateUpWeight) ? gateUpWeight.buffer : gateUpWeight);
636
1070
  }
637
1071
 
@@ -643,28 +1077,104 @@ export async function runDenseFFNWithFusedPostNormGPU(
643
1077
  }, recorder);
644
1078
 
645
1079
  if (recorder) {
1080
+ if (gateUpInputOwned) {
1081
+ recorder.trackTemporaryBuffer(gateUpInput.buffer);
1082
+ }
646
1083
  recorder.trackTemporaryBuffer(gateUpOutput.buffer);
647
1084
  } else {
1085
+ if (gateUpInputOwned) {
1086
+ releaseBuffer(gateUpInput.buffer);
1087
+ }
648
1088
  releaseBuffer(gateUpOutput.buffer);
649
1089
  }
650
1090
  } else {
651
- const gateWeight = getWeightBuffer(layerWeights.gate, 'ffn_gate');
652
- const upWeight = getWeightBuffer(layerWeights.up, 'ffn_up');
653
- const gateDtype = isWeightBuffer(layerWeights.gate) ? layerWeights.gate.dtype : 'f32';
654
- const upDtype = isWeightBuffer(layerWeights.up) ? layerWeights.up.dtype : 'f32';
1091
+ const hiddenSizeAligned32 = hiddenSize % 32 === 0;
1092
+ const activationDtype = selectRuleValue('shared', 'dtype', 'f16OrFallbackByFlag', {
1093
+ useF16,
1094
+ fallback: inputTensor.dtype,
1095
+ });
1096
+ const fusedGateUpWeights = resolveFusedGateUpWeights(layerWeights, {
1097
+ activationDtype,
1098
+ hiddenSize,
1099
+ kernelPath,
1100
+ phase,
1101
+ layerIdx,
1102
+ });
1103
+ const fusedGateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
1104
+ const fusedUpWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
1105
+ const gateDtype = fusedGateUpWeights.gateDtype ?? 'f32';
1106
+ const upDtype = fusedGateUpWeights.upDtype ?? 'f32';
655
1107
  const hasLoRAGate = Boolean(getLoRAModule(lora, layerIdx, 'gate_proj'));
656
1108
  const hasLoRAUp = Boolean(getLoRAModule(lora, layerIdx, 'up_proj'));
657
1109
  const dtypeMatches = gateDtype != null && upDtype != null && gateDtype === upDtype;
658
1110
  const q4kFusedAllowed = gateDtype !== 'q4k' || !isFusedQ4KDisabled({ kernelPath });
659
1111
  const dtypeSupported = gateDtype === 'f16' || gateDtype === 'f32' || (gateDtype === 'q4k' && q4kFusedAllowed);
660
- const canUseFusedGateUp = !hasLoRAGate && !hasLoRAUp && dtypeMatches && dtypeSupported;
1112
+ const canUseFusedGateUpByRule = selectRuleValue('inference', 'ffn', 'useFusedGateUp', {
1113
+ hasGate: true,
1114
+ hasUp: true,
1115
+ hasDown: true,
1116
+ hasFusedWeights: false,
1117
+ inputIsSupported: inputTensor.dtype === 'f32' || inputTensor.dtype === 'f16',
1118
+ hasLoRA: hasLoRAGate || hasLoRAUp,
1119
+ dtypeMatches,
1120
+ dtypeSupported,
1121
+ weightDtype: gateDtype,
1122
+ hasQ4KMaterialization: fusedGateUpWeights.hasQ4KMaterialization,
1123
+ activationDtype,
1124
+ f16BatchSupported: getKernelCapabilities().hasF16,
1125
+ batchSize: numTokens,
1126
+ hiddenSizeAligned32,
1127
+ });
1128
+ const canUseFusedGateUp = gateUpPathMode === 'split'
1129
+ ? false
1130
+ : canUseFusedGateUpByRule;
1131
+ trace.ffn(
1132
+ layerIdx,
1133
+ `useFusedGateUpWithPostNorm=${canUseFusedGateUp} gateUpPathMode=${gateUpPathMode} inputDtype=${inputTensor.dtype} activationDtype=${activationDtype} ` +
1134
+ `gateDtype=${gateDtype} upDtype=${upDtype} hasQ4KMaterialization=${fusedGateUpWeights.hasQ4KMaterialization} ` +
1135
+ `dtypeMatches=${dtypeMatches} dtypeSupported=${dtypeSupported} hiddenSizeAligned32=${hiddenSizeAligned32} batchSize=${numTokens}`
1136
+ );
1137
+ const gateWeight = canUseFusedGateUp
1138
+ ? fusedGateWeight
1139
+ : getWeightBuffer(layerWeights.gate, 'ffn_gate');
1140
+ const upWeight = canUseFusedGateUp
1141
+ ? fusedUpWeight
1142
+ : getWeightBuffer(layerWeights.up, 'ffn_up');
661
1143
 
662
1144
  if (canUseFusedGateUp) {
1145
+ const {
1146
+ fusedGateUpInputDtype,
1147
+ } = resolveDenseFFNFusedPathDtypes({
1148
+ phase,
1149
+ layerIdx,
1150
+ kernelPath,
1151
+ ffnStepPrecision,
1152
+ fallbackInputDtype: inputTensor.dtype,
1153
+ fallbackOutputDtype: matmulOutputDtype,
1154
+ });
1155
+ let fusedInput = inputTensor;
1156
+ let fusedInputOwned = false;
1157
+ if (fusedGateUpInputDtype && fusedGateUpInputDtype !== inputTensor.dtype) {
1158
+ fusedInput = await coerceTensorDtype(inputTensor, fusedGateUpInputDtype, recorder, {
1159
+ executionPolicies: context.executionPolicies ?? null,
1160
+ op: 'ffn_gate_up_input',
1161
+ transitionDeclaredBy: 'step_precision',
1162
+ });
1163
+ fusedInputOwned = fusedInput !== inputTensor;
1164
+ }
663
1165
  activatedOutput = await dispatchFusedGateUp({
664
- inputTensor, gateWeight, upWeight, gateDtype,
1166
+ inputTensor: fusedInput, gateWeight, upWeight, gateDtype,
665
1167
  hiddenSize, intermediateSize, numTokens,
666
1168
  hiddenActivation, swigluLimit, recorder,
1169
+ executionPolicies: context.executionPolicies ?? null,
667
1170
  });
1171
+ if (fusedInputOwned) {
1172
+ if (recorder) {
1173
+ recorder.trackTemporaryBuffer(fusedInput.buffer);
1174
+ } else {
1175
+ releaseBuffer(fusedInput.buffer);
1176
+ }
1177
+ }
668
1178
  } else {
669
1179
  const gateOutput = await doMatmul(
670
1180
  inputTensor, gateWeight,
@@ -676,6 +1186,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
676
1186
  label: `L${layerIdx}.ffn_gate`,
677
1187
  layerIdx,
678
1188
  kernelPath,
1189
+ executionPolicies: context.executionPolicies ?? null,
679
1190
  },
680
1191
  recorder
681
1192
  );
@@ -690,6 +1201,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
690
1201
  label: `L${layerIdx}.ffn_up`,
691
1202
  layerIdx,
692
1203
  kernelPath,
1204
+ executionPolicies: context.executionPolicies ?? null,
693
1205
  },
694
1206
  recorder
695
1207
  );
@@ -709,10 +1221,10 @@ export async function runDenseFFNWithFusedPostNormGPU(
709
1221
  }
710
1222
  }
711
1223
 
712
- if (!(layerWeights.gate instanceof GPUBuffer) && !isWeightBuffer(layerWeights.gate)) {
1224
+ if (!isGpuBufferInstance(layerWeights.gate) && !isWeightBuffer(layerWeights.gate)) {
713
1225
  releaseOrTrack(recorder, isWeightBuffer(gateWeight) ? gateWeight.buffer : gateWeight);
714
1226
  }
715
- if (!(layerWeights.up instanceof GPUBuffer) && !isWeightBuffer(layerWeights.up)) {
1227
+ if (!isGpuBufferInstance(layerWeights.up) && !isWeightBuffer(layerWeights.up)) {
716
1228
  releaseOrTrack(recorder, isWeightBuffer(upWeight) ? upWeight.buffer : upWeight);
717
1229
  }
718
1230
  }
@@ -739,10 +1251,10 @@ export async function runDenseFFNWithFusedPostNormGPU(
739
1251
  log.warn('Layer', `L${layerIdx} LoRA down_proj with fused kernel not yet optimized`);
740
1252
  }
741
1253
 
742
- if (!(layerWeights.down instanceof GPUBuffer) && !isWeightBuffer(layerWeights.down)) {
1254
+ if (!isGpuBufferInstance(layerWeights.down) && !isWeightBuffer(layerWeights.down)) {
743
1255
  releaseOrTrack(recorder, isWeightBuffer(downWeight) ? downWeight.buffer : downWeight);
744
1256
  }
745
- if (!(layerWeights.postFeedforwardNorm instanceof GPUBuffer)) releaseOrTrack(recorder, normWeightBuf);
1257
+ if (!isGpuBufferInstance(layerWeights.postFeedforwardNorm) && !isWeightBuffer(layerWeights.postFeedforwardNorm)) releaseOrTrack(recorder, normWeightBuf);
746
1258
  if (recorder) {
747
1259
  recorder.trackTemporaryBuffer(activatedOutput.buffer);
748
1260
  } else {