doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -4,7 +4,10 @@ import {
4
4
  buildLayerPipelineFromExecution,
5
5
  buildSessionRuntimePatch,
6
6
  PIPELINE_COMPATIBLE_OPS,
7
+ normalizeDtype,
8
+ requireSessionActivationDtype,
7
9
  } from './execution-runtime-builders.js';
10
+ import { mergeKernelPathPolicy } from '../../../config/merge-helpers.js';
8
11
  import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
9
12
  import { buildOpIdFromExecutionStep } from './operator-identity.js';
10
13
  import {
@@ -12,6 +15,7 @@ import {
12
15
  resolveFinitenessFallbackTransform,
13
16
  } from '../../../config/transforms/capability-transform-resolver.js';
14
17
  import { composeTransforms } from '../../../config/transforms/execution-graph-transforms.js';
18
+ import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
15
19
  import { log } from '../../../debug/index.js';
16
20
 
17
21
  export function hasExecutionV1(manifestInference) {
@@ -20,6 +24,308 @@ export function hasExecutionV1(manifestInference) {
20
24
  && typeof manifestInference.execution.kernels === 'object';
21
25
  }
22
26
 
27
+ function mergeExecutionV1Session(manifestSession, runtimeSession) {
28
+ return mergeRuntimeValues(
29
+ manifestSession ?? {},
30
+ runtimeSession ?? {}
31
+ );
32
+ }
33
+
34
+ function hasOwnProperty(value, key) {
35
+ return value != null && Object.prototype.hasOwnProperty.call(value, key);
36
+ }
37
+
38
+ function resolveRuntimeInferenceOverrideSection(runtimeOverrides, key) {
39
+ const inferenceOverrides = runtimeOverrides?.inference;
40
+ if (!inferenceOverrides || typeof inferenceOverrides !== 'object' || Array.isArray(inferenceOverrides)) {
41
+ return null;
42
+ }
43
+ if (!hasOwnProperty(inferenceOverrides, key)) {
44
+ return null;
45
+ }
46
+ return inferenceOverrides[key] ?? null;
47
+ }
48
+
49
+ function applyExecutionV1RuntimeComputeOverrides(session, runtimeSession, runtimeCompute) {
50
+ const activationOverrideRaw = runtimeCompute?.activationDtype;
51
+ if (activationOverrideRaw == null) {
52
+ return session;
53
+ }
54
+
55
+ const activationOverride = normalizeDtype(
56
+ activationOverrideRaw,
57
+ 'runtime.inference.compute.activationDtype'
58
+ );
59
+ const runtimeSessionActivation = runtimeSession?.compute?.defaults?.activationDtype;
60
+ if (
61
+ runtimeSessionActivation != null
62
+ && normalizeDtype(
63
+ runtimeSessionActivation,
64
+ 'runtime.inference.session.compute.defaults.activationDtype'
65
+ ) !== activationOverride
66
+ ) {
67
+ throw new Error(
68
+ '[ExecutionV1] runtime.inference.compute.activationDtype conflicts with ' +
69
+ 'runtime.inference.session.compute.defaults.activationDtype. ' +
70
+ 'Set both to the same dtype or remove one override.'
71
+ );
72
+ }
73
+
74
+ const runtimeSessionOutput = runtimeSession?.compute?.defaults?.outputDtype;
75
+ if (
76
+ runtimeSessionOutput != null
77
+ && normalizeDtype(
78
+ runtimeSessionOutput,
79
+ 'runtime.inference.session.compute.defaults.outputDtype'
80
+ ) !== activationOverride
81
+ ) {
82
+ throw new Error(
83
+ '[ExecutionV1] runtime.inference.compute.activationDtype conflicts with ' +
84
+ 'runtime.inference.session.compute.defaults.outputDtype. ' +
85
+ 'Set both to the same dtype or remove one override.'
86
+ );
87
+ }
88
+
89
+ return {
90
+ ...session,
91
+ compute: {
92
+ ...(session?.compute ?? {}),
93
+ defaults: {
94
+ ...(session?.compute?.defaults ?? {}),
95
+ activationDtype: activationOverride,
96
+ outputDtype: activationOverride,
97
+ },
98
+ },
99
+ };
100
+ }
101
+
102
+ const EXECUTION_V1_PROJECTION_OPS = new Set([
103
+ 'q_proj', 'k_proj', 'v_proj', 'o_proj',
104
+ 'gate_proj', 'up_proj', 'down_proj',
105
+ ]);
106
+
107
+ const EXECUTION_V1_DENSE_Q4_PREFILL_FILES = new Set([
108
+ 'matmul_f16w_f32a.wgsl',
109
+ 'matmul_f16w_f32a_tiled.wgsl',
110
+ 'matmul_f16.wgsl',
111
+ 'matmul_f16_tiled.wgsl',
112
+ ]);
113
+
114
+ const EXECUTION_V1_F32_ACTIVATION_NARROWING_FILES = new Set([
115
+ 'rmsnorm.wgsl',
116
+ 'rope.wgsl',
117
+ 'residual.wgsl',
118
+ 'gelu.wgsl',
119
+ 'sample.wgsl',
120
+ 'gather.wgsl',
121
+ 'matmul_gemv_subgroup.wgsl',
122
+ 'matmul_f16w_f32a.wgsl',
123
+ 'matmul_f16w_f32a_tiled.wgsl',
124
+ 'attention_decode_online_f16kv.wgsl',
125
+ 'attention_decode_chunked_f16kv.wgsl',
126
+ 'attention_small_f16kv.wgsl',
127
+ 'attention_streaming_f16kv.wgsl',
128
+ 'attention_decode.wgsl',
129
+ 'attention_small.wgsl',
130
+ 'attention_streaming.wgsl',
131
+ 'silu.wgsl',
132
+ ]);
133
+
134
+ function summarizeExecutionGraphContext(execution) {
135
+ const summary = {
136
+ hasDensePrefillProjectionKernel: false,
137
+ hasQ4DecodeProjectionKernel: false,
138
+ hasQ4PrefillProjectionKernel: false,
139
+ hasAvailableQ4PrefillProjectionKernel: false,
140
+ requiresF16ActivationNarrowing: false,
141
+ };
142
+ const phases = [
143
+ ['decode', execution?.decode ?? []],
144
+ ['prefill', execution?.prefill ?? []],
145
+ ];
146
+
147
+ for (const [phase, steps] of phases) {
148
+ for (const step of steps) {
149
+ if (!EXECUTION_V1_PROJECTION_OPS.has(step[0])) {
150
+ continue;
151
+ }
152
+ const kernelEntry = execution?.kernels?.[step[1]];
153
+ if (!kernelEntry) {
154
+ continue;
155
+ }
156
+ if (phase === 'decode' && kernelEntry.kernel === 'fused_matmul_q4.wgsl') {
157
+ summary.hasQ4DecodeProjectionKernel = true;
158
+ }
159
+ if (phase === 'prefill' && EXECUTION_V1_DENSE_Q4_PREFILL_FILES.has(kernelEntry.kernel)) {
160
+ summary.hasDensePrefillProjectionKernel = true;
161
+ }
162
+ if (phase === 'prefill' && kernelEntry.kernel.startsWith('fused_matmul_q4')) {
163
+ summary.hasQ4PrefillProjectionKernel = true;
164
+ }
165
+ }
166
+ }
167
+
168
+ summary.hasAvailableQ4PrefillProjectionKernel = Object.values(execution?.kernels ?? {}).some(
169
+ (entry) => entry?.kernel === 'fused_matmul_q4_batched_multicol_shared.wgsl'
170
+ || entry?.kernel === 'fused_matmul_q4_batched.wgsl'
171
+ );
172
+ summary.requiresF16ActivationNarrowing = Object.values(execution?.kernels ?? {}).some(
173
+ (entry) => EXECUTION_V1_F32_ACTIVATION_NARROWING_FILES.has(entry?.kernel)
174
+ );
175
+
176
+ return summary;
177
+ }
178
+
179
+ function normalizeExecutionLayerType(layerType) {
180
+ return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
181
+ }
182
+
183
+ function isLinearExecutionLayerType(layerType) {
184
+ const normalized = normalizeExecutionLayerType(layerType);
185
+ return normalized === 'linear_attention'
186
+ || normalized === 'linear'
187
+ || normalized === 'gated_delta'
188
+ || normalized === 'gated_delta_net';
189
+ }
190
+
191
+ function isFullAttentionExecutionLayerType(layerType) {
192
+ const normalized = normalizeExecutionLayerType(layerType);
193
+ return normalized === 'full_attention'
194
+ || normalized === 'full'
195
+ || normalized === 'global'
196
+ || normalized === 'standard';
197
+ }
198
+
199
+ function isFusedQ4ProjectionKernel(kernelEntry) {
200
+ return typeof kernelEntry?.kernel === 'string'
201
+ && kernelEntry.kernel.startsWith('fused_matmul_q4');
202
+ }
203
+
204
+ function collectPhaseOpEntries(phaseEntries, kernels, ops) {
205
+ const entries = [];
206
+ for (const entry of phaseEntries ?? []) {
207
+ if (Array.isArray(entry)) {
208
+ if (!ops.has(entry[0])) {
209
+ continue;
210
+ }
211
+ entries.push({
212
+ op: entry[0],
213
+ layers: 'all',
214
+ kernelEntry: kernels?.[entry[1]] ?? null,
215
+ });
216
+ continue;
217
+ }
218
+ if (!entry || typeof entry !== 'object' || !Array.isArray(entry.layers) || !Array.isArray(entry.steps)) {
219
+ continue;
220
+ }
221
+ for (const step of entry.steps) {
222
+ if (!Array.isArray(step) || !ops.has(step[0])) {
223
+ continue;
224
+ }
225
+ entries.push({
226
+ op: step[0],
227
+ layers: entry.layers,
228
+ kernelEntry: kernels?.[step[1]] ?? null,
229
+ });
230
+ }
231
+ }
232
+ return entries;
233
+ }
234
+
235
+ function collectCoveredLayers(entries, targetLayers, predicate) {
236
+ const covered = new Set();
237
+ for (const entry of entries) {
238
+ if (!predicate(entry.kernelEntry)) {
239
+ continue;
240
+ }
241
+ if (entry.layers === 'all') {
242
+ for (const layerIdx of targetLayers) {
243
+ covered.add(layerIdx);
244
+ }
245
+ continue;
246
+ }
247
+ for (const layerIdx of entry.layers ?? []) {
248
+ if (targetLayers.includes(layerIdx)) {
249
+ covered.add(layerIdx);
250
+ }
251
+ }
252
+ }
253
+ return covered;
254
+ }
255
+
256
+ function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId) {
257
+ if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
258
+ return;
259
+ }
260
+
261
+ const linearLayers = layerTypes
262
+ .map((layerType, layerIdx) => ({ layerType, layerIdx }))
263
+ .filter(({ layerType }) => isLinearExecutionLayerType(layerType))
264
+ .map(({ layerIdx }) => layerIdx);
265
+ const fullAttentionLayers = layerTypes
266
+ .map((layerType, layerIdx) => ({ layerType, layerIdx }))
267
+ .filter(({ layerType }) => isFullAttentionExecutionLayerType(layerType))
268
+ .map(({ layerIdx }) => layerIdx);
269
+
270
+ if (linearLayers.length === 0 || fullAttentionLayers.length === 0) {
271
+ return;
272
+ }
273
+
274
+ const guardedOps = [
275
+ {
276
+ phase: 'decode',
277
+ label: 'q_proj/linear_qkv_proj',
278
+ ops: new Set(['q_proj', 'qkv_proj', 'linear_qkv_proj']),
279
+ },
280
+ {
281
+ phase: 'decode',
282
+ label: 'o_proj/linear_out_proj',
283
+ ops: new Set(['o_proj', 'linear_out_proj']),
284
+ },
285
+ {
286
+ phase: 'prefill',
287
+ label: 'q_proj/linear_qkv_proj',
288
+ ops: new Set(['q_proj', 'qkv_proj', 'linear_qkv_proj']),
289
+ },
290
+ {
291
+ phase: 'prefill',
292
+ label: 'o_proj/linear_out_proj',
293
+ ops: new Set(['o_proj', 'linear_out_proj']),
294
+ },
295
+ ];
296
+
297
+ for (const guard of guardedOps) {
298
+ const entries = collectPhaseOpEntries(execution?.[guard.phase], execution?.kernels, guard.ops);
299
+ if (entries.length === 0) {
300
+ continue;
301
+ }
302
+
303
+ const hasUnsafeGlobalEntry = entries.some(
304
+ (entry) => entry.layers === 'all' && !isFusedQ4ProjectionKernel(entry.kernelEntry)
305
+ );
306
+ if (!hasUnsafeGlobalEntry) {
307
+ continue;
308
+ }
309
+
310
+ const coveredLinearLayers = collectCoveredLayers(
311
+ entries,
312
+ linearLayers,
313
+ (kernelEntry) => isFusedQ4ProjectionKernel(kernelEntry)
314
+ );
315
+ const missingLinearLayers = linearLayers.filter((layerIdx) => !coveredLinearLayers.has(layerIdx));
316
+ if (missingLinearLayers.length === 0) {
317
+ continue;
318
+ }
319
+
320
+ throw new Error(
321
+ `[ExecutionV1] Hybrid linear-attention model "${modelId}" cannot apply a global non-Q4 ` +
322
+ `${guard.phase} ${guard.label} kernel because linear-attention layers alias that role. ` +
323
+ `Missing fused Q4 coverage for linear layers [${missingLinearLayers.join(', ')}]. ` +
324
+ 'Keep the linear-attention path on fused_matmul_q4* or isolate those layers with explicit execution-v1 overrides.'
325
+ );
326
+ }
327
+ }
328
+
23
329
 
24
330
  function expandV1ToResolvedSteps(execution, options = {}) {
25
331
  const expanded = expandExecutionV1(execution, options);
@@ -35,6 +341,7 @@ function expandV1ToResolvedSteps(execution, options = {}) {
35
341
  entry: step.entry,
36
342
  ...(step.weights ? { weights: step.weights } : {}),
37
343
  ...(step.constants ? { constants: step.constants } : {}),
344
+ ...(step.precision ? { precision: step.precision } : {}),
38
345
  layers: step.layers,
39
346
  kernelRef: {
40
347
  id: `${step.kernel.replace('.wgsl', '')}.${step.entry}`,
@@ -52,22 +359,36 @@ export function compileExecutionV1(options = {}) {
52
359
  const manifestInference = options.manifestInference;
53
360
  const modelId = options.modelId ?? 'model';
54
361
  const numLayers = options.numLayers ?? 0;
362
+ const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
55
363
  const capabilities = options.capabilities ?? null;
56
364
  const platform = options.platform ?? null;
365
+ const runtimeSession = options.runtimeSession ?? null;
366
+ const runtimeCompute = options.runtimeCompute ?? null;
367
+ const kernelPathPolicy = mergeKernelPathPolicy(undefined, options.kernelPathPolicy ?? undefined);
57
368
 
58
369
  if (!hasExecutionV1(manifestInference)) {
59
370
  throw new Error(`[ExecutionV1] manifest.inference.schema must be "${EXECUTION_V1_SCHEMA_ID}".`);
60
371
  }
61
372
 
62
373
  let execution = manifestInference.execution;
63
- const sessionDefaults = manifestInference.sessionDefaults;
374
+ const mergedSession = mergeExecutionV1Session(
375
+ manifestInference.session ?? {},
376
+ runtimeSession ?? {}
377
+ );
378
+ const session = applyExecutionV1RuntimeComputeOverrides(
379
+ mergedSession,
380
+ runtimeSession,
381
+ runtimeCompute
382
+ );
64
383
 
65
- if (!sessionDefaults?.compute?.defaults?.activationDtype) {
66
- throw new Error('[ExecutionV1] sessionDefaults.compute.defaults.activationDtype is required.');
384
+ if (!session?.compute?.defaults?.activationDtype) {
385
+ throw new Error('[ExecutionV1] session.compute.defaults.activationDtype is required.');
67
386
  }
68
387
 
69
- const activationDtype = sessionDefaults.compute.defaults.activationDtype;
70
- const kvDtype = sessionDefaults?.kvcache?.kvDtype ?? activationDtype;
388
+ const activationDtype = session.compute.defaults.activationDtype;
389
+ const kvDtype = session?.kvcache?.kvDtype ?? activationDtype;
390
+ const layerTypes = manifestInference?.layerPattern?.layerTypes ?? null;
391
+ const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(runtimeCompute);
71
392
 
72
393
  // Validate the original manifest graph (full digest checks).
73
394
  expandExecutionV1(execution);
@@ -77,19 +398,40 @@ export function compileExecutionV1(options = {}) {
77
398
  // -------------------------------------------------------------------------
78
399
  let appliedTransformNames = [];
79
400
  let fallbackExecution = null;
401
+ let fallbackTransformResult = null;
80
402
  let graphWasTransformed = false;
81
403
 
82
404
  if (capabilities) {
83
- const graphContext = { activationDtype, kvDtype };
405
+ const graphContext = {
406
+ activationDtype,
407
+ kvDtype,
408
+ headDim,
409
+ modelId,
410
+ layerTypes,
411
+ ...summarizeExecutionGraphContext(execution),
412
+ };
84
413
  const resolved = resolveCapabilityTransforms(capabilities, platform, graphContext);
414
+ const sourceScope = kernelPathPolicy.sourceScope ?? kernelPathPolicy.allowSources ?? [];
415
+ const remapAllowed = kernelPathPolicy.mode === 'capability-aware'
416
+ && kernelPathPolicy.onIncompatible === 'remap'
417
+ && sourceScope.includes('manifest');
85
418
 
86
419
  if (resolved.transforms.length > 0) {
420
+ if (!remapAllowed) {
421
+ throw new Error(
422
+ `[ExecutionV1] capability transforms required for "${modelId}" (${resolved.reason}) ` +
423
+ `but runtime.inference.kernelPathPolicy is ${JSON.stringify(kernelPathPolicy)}. ` +
424
+ 'Use capability-aware remap for manifest-owned execution graphs or choose a compatible runtime.'
425
+ );
426
+ }
87
427
  const composed = composeTransforms(...resolved.transforms);
88
428
  const transformed = composed(execution, {
89
429
  capabilities,
90
430
  platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
91
431
  activationDtype,
92
432
  kvDtype,
433
+ modelId,
434
+ layerTypes,
93
435
  });
94
436
  if (transformed !== execution) {
95
437
  execution = transformed;
@@ -104,25 +446,32 @@ export function compileExecutionV1(options = {}) {
104
446
  log.debug('ExecutionV1', `No capability transforms needed (${resolved.reason})`);
105
447
  }
106
448
 
107
- // Build finiteness fallback from the (possibly transformed) graph
108
- const fallbackResult = resolveFinitenessFallbackTransform(graphContext);
109
- if (fallbackResult) {
110
- const fallbackGraph = fallbackResult.transform(execution, {
449
+ // Build explicit finiteness fallback only when the runtime opted into
450
+ // alternate-plan recovery. The default policy is fail-fast.
451
+ fallbackTransformResult = finitenessPolicy.onTrigger === 'fallback-plan'
452
+ ? resolveFinitenessFallbackTransform(graphContext)
453
+ : null;
454
+ if (fallbackTransformResult) {
455
+ const fallbackGraph = fallbackTransformResult.transform(execution, {
111
456
  capabilities,
112
457
  platform: platform ?? { id: 'unknown', vendor: 'unknown', architecture: 'unknown' },
113
458
  activationDtype,
114
459
  kvDtype,
460
+ headDim,
461
+ modelId,
115
462
  });
116
463
  if (fallbackGraph) {
117
464
  fallbackExecution = fallbackGraph;
118
465
  log.info(
119
466
  'ExecutionV1',
120
- `Finiteness fallback transform resolved: ${fallbackResult.name}`
467
+ `Finiteness fallback transform resolved: ${fallbackTransformResult.name}`
121
468
  );
122
469
  }
123
470
  }
124
471
  }
125
472
 
473
+ assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
474
+
126
475
  // Transformed graphs have null digests for derived kernels — skip digest
127
476
  // validation since the original graph was already validated above.
128
477
  const expandOptions = graphWasTransformed ? { skipDigestValidation: true } : {};
@@ -132,34 +481,38 @@ export function compileExecutionV1(options = {}) {
132
481
  const decodeSteps = resolvedSteps.filter((s) => s.phase === 'decode' || s.phase === 'both');
133
482
 
134
483
  // When widenToF32Activations was applied, the graph's kernels now expect f32
135
- // activations. The sessionDefaults must reflect this for kernel path building
484
+ // activations. The resolved session must reflect this for kernel path building
136
485
  // and the runtime session patch. When the GPU has no f16 at all (full f32),
137
486
  // KV cache and all compute dtypes must also be f32.
138
487
  const activationWidened = appliedTransformNames.includes('widenToF32Activations');
139
488
  const fullF32 = activationWidened && capabilities?.hasF16 === false;
140
- let effectiveSessionDefaults = sessionDefaults;
141
- if (activationWidened) {
489
+ const useQwenSelectiveF16Primary =
490
+ appliedTransformNames.includes('useQwenF16PrimaryMatmuls')
491
+ && !appliedTransformNames.includes('narrowToF16Activations');
492
+ let effectiveSession = session;
493
+ if (activationWidened || useQwenSelectiveF16Primary) {
142
494
  const f32Defaults = fullF32
143
495
  ? { activationDtype: 'f32', mathDtype: 'f32', accumDtype: 'f32', outputDtype: 'f32' }
144
- : { activationDtype: 'f32' };
145
- effectiveSessionDefaults = {
146
- ...sessionDefaults,
496
+ : useQwenSelectiveF16Primary
497
+ ? { activationDtype: 'f32', outputDtype: 'f32' }
498
+ : { activationDtype: 'f32' };
499
+ effectiveSession = {
500
+ ...session,
147
501
  compute: {
148
- ...sessionDefaults.compute,
502
+ ...session.compute,
149
503
  defaults: {
150
- ...sessionDefaults.compute.defaults,
504
+ ...session.compute.defaults,
151
505
  ...f32Defaults,
152
506
  },
153
507
  },
154
- ...(fullF32 && sessionDefaults?.kvcache ? {
508
+ ...(fullF32 && session?.kvcache ? {
155
509
  kvcache: {
156
- ...sessionDefaults.kvcache,
510
+ ...session.kvcache,
157
511
  kvDtype: 'f32',
158
512
  },
159
513
  } : {}),
160
514
  };
161
515
  }
162
-
163
516
  const inlineKernelPathEnabled = execution.inlineKernelPath !== false;
164
517
  const finitenessFallback = typeof execution.finitenessFallbackKernelPathId === 'string'
165
518
  && execution.finitenessFallbackKernelPathId.length > 0
@@ -168,7 +521,7 @@ export function compileExecutionV1(options = {}) {
168
521
  const kernelPath = inlineKernelPathEnabled
169
522
  ? buildInlineKernelPath(
170
523
  resolvedSteps,
171
- effectiveSessionDefaults,
524
+ effectiveSession,
172
525
  modelId,
173
526
  numLayers,
174
527
  finitenessFallback
@@ -179,24 +532,37 @@ export function compileExecutionV1(options = {}) {
179
532
  let fallbackKernelPath = null;
180
533
  if (fallbackExecution && inlineKernelPathEnabled) {
181
534
  const fallbackSteps = expandV1ToResolvedSteps(fallbackExecution, { skipDigestValidation: true });
182
- const fallbackSessionDefaults = {
183
- ...sessionDefaults,
535
+ const fallbackKvDtype = fallbackTransformResult?.fallbackKvDtype ?? 'f32';
536
+ const fallbackSession = {
537
+ ...session,
184
538
  compute: {
185
- ...sessionDefaults.compute,
539
+ ...session.compute,
186
540
  defaults: {
187
- ...sessionDefaults.compute.defaults,
541
+ ...session.compute.defaults,
188
542
  activationDtype: 'f32',
543
+ mathDtype: 'f32',
544
+ accumDtype: 'f32',
545
+ outputDtype: 'f32',
189
546
  },
190
547
  },
548
+ kvcache: {
549
+ ...(session.kvcache ?? {}),
550
+ kvDtype: fallbackKvDtype,
551
+ },
191
552
  };
192
553
  fallbackKernelPath = buildInlineKernelPath(
193
554
  fallbackSteps,
194
- fallbackSessionDefaults,
555
+ fallbackSession,
195
556
  modelId,
196
557
  numLayers,
197
558
  null
198
559
  );
199
560
  if (fallbackKernelPath) {
561
+ fallbackKernelPath = {
562
+ ...fallbackKernelPath,
563
+ id: `${fallbackKernelPath.id}-finiteness-fallback`,
564
+ name: 'Execution inline kernel path (finiteness fallback)',
565
+ };
200
566
  log.info(
201
567
  'ExecutionV1',
202
568
  `Finiteness fallback inline kernel path built (${fallbackKernelPath.id})`
@@ -204,7 +570,10 @@ export function compileExecutionV1(options = {}) {
204
570
  }
205
571
  }
206
572
 
207
- const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps);
573
+ const layerPipelineResult = buildLayerPipelineFromExecution(resolvedSteps, {
574
+ logIncompatibleOps: !(kernelPath && inlineKernelPathEnabled),
575
+ ffnDtypeFallback: requireSessionActivationDtype(effectiveSession),
576
+ });
208
577
  if (layerPipelineResult?.incompatibleOps && !kernelPath && inlineKernelPathEnabled) {
209
578
  throw new Error(
210
579
  `[ExecutionV1] execution contains layer ops not compatible with the JS layer pipeline ` +
@@ -213,12 +582,12 @@ export function compileExecutionV1(options = {}) {
213
582
  );
214
583
  }
215
584
  const layerPipeline = layerPipelineResult?.incompatibleOps ? null : layerPipelineResult;
216
- const sessionPatch = buildSessionRuntimePatch(effectiveSessionDefaults, {
585
+ const sessionPatch = buildSessionRuntimePatch(effectiveSession, {
217
586
  includeDecodeLoop: false,
218
587
  });
219
588
 
220
589
  return {
221
- sessionDefaults: effectiveSessionDefaults,
590
+ session: effectiveSession,
222
591
  policies: execution.policies,
223
592
  resolvedSteps: {
224
593
  prefill: prefillSteps,
@@ -240,8 +609,8 @@ export function compileExecutionV1(options = {}) {
240
609
  // 1. compileExecutionV1 — resolves execution graph, applies capability transforms,
241
610
  // builds inline kernel path and layer pipeline from the execution graph.
242
611
  // 2. mergeRuntimeValues — merges the runtimeInferencePatch into runtimeConfig.inference.
243
- // This writes kernelPath, kernelPathSource, pipeline, compute, kvcache, and
244
- // session defaults into the runtime config. decodeLoop stays manifest-owned
612
+ // This writes kernelPath, kernelPathSource, pipeline, compute, and session
613
+ // into the runtime config. decodeLoop stays manifest-owned
245
614
  // until applyModelBatchingRuntimeDefaults in phase 2. If runtime batching was
246
615
  // already explicitly configured, manifest decodeLoop is skipped and runtime
247
616
  // values take precedence.
@@ -251,6 +620,7 @@ export function compileExecutionV1(options = {}) {
251
620
  // produce incorrect results.
252
621
  export function applyExecutionV1RuntimeConfig(options = {}) {
253
622
  const runtimeConfig = options.runtimeConfig ?? null;
623
+ const runtimeOverrides = options.runtimeOverrides;
254
624
  const manifest = options.manifest ?? null;
255
625
  if (!runtimeConfig || !manifest?.inference) {
256
626
  return { runtimeConfig, executionV1State: null };
@@ -271,8 +641,16 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
271
641
  manifestInference: manifest.inference,
272
642
  modelId: manifest.modelId ?? options.modelId,
273
643
  numLayers: options.numLayers ?? manifest.architecture?.numLayers ?? 0,
644
+ headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
274
645
  capabilities: options.capabilities ?? null,
275
646
  platform: options.platform ?? null,
647
+ runtimeSession: hasOwnProperty(options, 'runtimeOverrides')
648
+ ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'session')
649
+ : (runtimeConfig.inference?.session ?? null),
650
+ runtimeCompute: hasOwnProperty(options, 'runtimeOverrides')
651
+ ? resolveRuntimeInferenceOverrideSection(runtimeOverrides, 'compute')
652
+ : (runtimeConfig.inference?.compute ?? null),
653
+ kernelPathPolicy: runtimeConfig.inference?.kernelPathPolicy ?? null,
276
654
  });
277
655
 
278
656
  const runtimeInferencePatch = executionV1State.runtimeInferencePatch;
@@ -10,8 +10,67 @@
10
10
  */
11
11
 
12
12
  import type { Tensor } from '../../../../gpu/tensor.js';
13
+ import type { WeightBuffer } from '../../../../gpu/weight-buffer.js';
13
14
  import type { LayerContext, LayerWeights } from '../types.js';
14
15
 
16
+ export declare function resolveGateUpPathMode(options?: {
17
+ kernelPath?: Record<string, unknown> | null;
18
+ phase?: 'prefill' | 'decode' | null;
19
+ layerIdx?: number;
20
+ }): 'fused' | 'split' | 'implicit';
21
+
22
+ export declare function resolveFusedGateUpWeights(
23
+ layerWeights: LayerWeights | undefined,
24
+ options?: {
25
+ activationDtype?: 'f16' | 'f32' | null;
26
+ hiddenSize?: number;
27
+ kernelPath?: Record<string, unknown> | null;
28
+ phase?: 'prefill' | 'decode' | null;
29
+ layerIdx?: number;
30
+ }
31
+ ): {
32
+ gate: GPUBuffer | WeightBuffer | null;
33
+ up: GPUBuffer | WeightBuffer | null;
34
+ gateDtype: string | null;
35
+ upDtype: string | null;
36
+ hasQ4KMaterialization: boolean;
37
+ };
38
+
39
+ export declare function resolveDenseFFNMatmulStepDtype(options?: {
40
+ role?: string | null;
41
+ phase?: 'prefill' | 'decode' | null;
42
+ layerIdx?: number;
43
+ kernelPath?: Record<string, unknown> | null;
44
+ fallback?: 'f16' | 'f32' | null;
45
+ field?: 'inputDtype' | 'outputDtype';
46
+ ffnStepPrecision?: {
47
+ inputDtype?: 'f16' | 'f32' | null;
48
+ outputDtype?: 'f16' | 'f32' | null;
49
+ } | null;
50
+ }): 'f16' | 'f32' | null;
51
+
52
+ export declare function resolveDenseFFNFusedPathDtypes(options?: {
53
+ phase?: 'prefill' | 'decode' | null;
54
+ layerIdx?: number;
55
+ kernelPath?: Record<string, unknown> | null;
56
+ fallbackInputDtype?: 'f16' | 'f32' | null;
57
+ fallbackOutputDtype?: 'f16' | 'f32' | null;
58
+ ffnStepPrecision?: {
59
+ inputDtype?: 'f16' | 'f32' | null;
60
+ outputDtype?: 'f16' | 'f32' | null;
61
+ } | null;
62
+ }): {
63
+ fusedGateUpInputDtype: 'f16' | 'f32' | null;
64
+ fusedGateUpOutputDtype: 'f16' | 'f32' | null;
65
+ downInputDtype: 'f16' | 'f32' | null;
66
+ };
67
+
68
+ export declare function canUseNativeF16FusedGateUp(options?: {
69
+ inputDtype?: 'f16' | 'f32' | null;
70
+ gateDtype?: string | null;
71
+ hasF16?: boolean;
72
+ }): boolean;
73
+
15
74
  /**
16
75
  * Run dense (non-MoE) FFN on GPU.
17
76
  */