doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,4 +1,5 @@
1
1
  import { selectRuleValue } from '../../../rules/rule-registry.js';
2
+ import { READBACK_MODES } from '../../../config/schema/execution-v1.schema.js';
2
3
  import {
3
4
  resolveDeferredRoundingWindowTokens,
4
5
  resolveRangeAwareSelectiveWideningConfig,
@@ -7,6 +8,7 @@ import { log } from '../../../debug/index.js';
7
8
 
8
9
  export const PRIMARY_EXECUTION_PLAN_ID = 'primary';
9
10
  export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
11
+ const DEFAULT_MAX_TOKENS = 256;
10
12
 
11
13
  function assertOptionalBoolean(value, label) {
12
14
  if (value === undefined) {
@@ -40,6 +42,15 @@ function assertOptionalStopCheckMode(value) {
40
42
  return value;
41
43
  }
42
44
 
45
+ function assertReadbackMode(value) {
46
+ if (!value || !READBACK_MODES.includes(value)) {
47
+ throw new Error(
48
+ `[ExecutionPlan] readbackMode must be one of ${READBACK_MODES.join(', ')}; got ${JSON.stringify(value)}.`
49
+ );
50
+ }
51
+ return value;
52
+ }
53
+
43
54
  function resolveFallbackActivationDtype(primaryActivationDtype) {
44
55
  const fallbackActivationDtype = selectRuleValue(
45
56
  'inference',
@@ -55,6 +66,43 @@ function resolveFallbackActivationDtype(primaryActivationDtype) {
55
66
  return fallbackActivationDtype;
56
67
  }
57
68
 
69
+ function kernelUsesF16ActivationExecution(kernel) {
70
+ const kernelName = String(kernel ?? '');
71
+ if (!kernelName) {
72
+ return false;
73
+ }
74
+ return /_f16(?!w|kv)/.test(kernelName);
75
+ }
76
+
77
+ function stepUsesF16ActivationExecution(step) {
78
+ if (!step || typeof step !== 'object') {
79
+ return false;
80
+ }
81
+ const precision = step.precision ?? null;
82
+ if (
83
+ precision?.activationDtype === 'f16'
84
+ || precision?.inputDtype === 'f16'
85
+ || precision?.outputDtype === 'f16'
86
+ ) {
87
+ return true;
88
+ }
89
+ return kernelUsesF16ActivationExecution(step.kernel);
90
+ }
91
+
92
+ function kernelPathUsesF16ActivationExecution(kernelPath) {
93
+ const stepLists = [
94
+ kernelPath?.decode?.steps,
95
+ kernelPath?.prefill?.steps,
96
+ kernelPath?.preLayer,
97
+ kernelPath?.postLayer,
98
+ kernelPath?.sampling,
99
+ ];
100
+ for (const override of kernelPath?.layerOverrides ?? []) {
101
+ stepLists.push(override?.steps, override?.decode?.steps, override?.prefill?.steps);
102
+ }
103
+ return stepLists.some((steps) => Array.isArray(steps) && steps.some((step) => stepUsesF16ActivationExecution(step)));
104
+ }
105
+
58
106
  function createStaticExecutionPlan({
59
107
  id,
60
108
  source,
@@ -74,15 +122,17 @@ function createStaticExecutionPlan({
74
122
  kernelPathSource,
75
123
  activationDtype,
76
124
  finitenessGuardEnabled: activationDtype === 'f16' && finitenessPolicy.enabled,
125
+ finitenessOnTrigger: finitenessPolicy.onTrigger,
77
126
  finitenessAbsThreshold: finitenessPolicy.absThreshold,
78
127
  finitenessIncludeNonFinite: finitenessPolicy.includeNonFinite,
79
128
  deferredRoundingWindowTokens,
80
- defaultDisableCommandBatching: generationConfig.disableCommandBatching,
129
+ defaultDisableCommandBatching: batchingConfig.disableCommandBatching,
81
130
  defaultDisableMultiTokenDecode: generationConfig.disableMultiTokenDecode,
82
131
  defaultBatchSize: batchingConfig.batchSize,
83
132
  defaultStopCheckMode: batchingConfig.stopCheckMode,
84
133
  defaultMaxTokens: batchingConfig.maxTokens,
85
134
  readbackInterval: batchingConfig.readbackInterval,
135
+ readbackMode: batchingConfig.readbackMode,
86
136
  ringTokens: batchingConfig.ringTokens,
87
137
  ringStop: batchingConfig.ringStop,
88
138
  ringStaging: batchingConfig.ringStaging,
@@ -114,7 +164,8 @@ function validatePlanAgainstManifest(plan, manifest) {
114
164
  const manifestInf = manifest.inference ?? manifest;
115
165
  const warnings = [];
116
166
 
117
- const manifestActivationDtype = manifestInf.compute?.activationDtype
167
+ const manifestActivationDtype = manifestInf.session?.compute?.defaults?.activationDtype
168
+ ?? manifestInf.compute?.activationDtype
118
169
  ?? manifestInf.quantizationInfo?.compute
119
170
  ?? null;
120
171
  if (manifestActivationDtype && plan.activationDtype
@@ -124,17 +175,6 @@ function validatePlanAgainstManifest(plan, manifest) {
124
175
  );
125
176
  }
126
177
 
127
- const manifestKernelPathId = manifestInf.execution?.kernelPathId
128
- ?? manifestInf.defaultKernelPath
129
- ?? null;
130
- if (manifestKernelPathId && plan.kernelPathId
131
- && plan.kernelPathId !== manifestKernelPathId
132
- && plan.kernelPathSource !== 'config') {
133
- warnings.push(
134
- `kernelPathId: plan="${plan.kernelPathId}" vs manifest="${manifestKernelPathId}"`
135
- );
136
- }
137
-
138
178
  if (warnings.length > 0) {
139
179
  log.warn(
140
180
  'ExecutionPlan',
@@ -157,11 +197,35 @@ export function compileExecutionPlanState(options) {
157
197
 
158
198
  const inferenceConfig = runtimeConfig.inference;
159
199
  const computeConfig = inferenceConfig.compute;
160
- const generationConfig = inferenceConfig.generation;
161
- const batchingConfig = inferenceConfig.batching;
200
+ const generationConfig = inferenceConfig.generation ?? {};
201
+ if (generationConfig.disableCommandBatching !== undefined) {
202
+ throw new Error(
203
+ '[ExecutionPlan] runtime.inference.generation.disableCommandBatching is removed. ' +
204
+ 'Use runtime.inference.session.decodeLoop.disableCommandBatching.'
205
+ );
206
+ }
207
+ const sessionConfig = inferenceConfig.session ?? {};
208
+ const decodeLoopConfig = sessionConfig.decodeLoop;
209
+ if (!decodeLoopConfig || typeof decodeLoopConfig !== 'object') {
210
+ throw new Error('[ExecutionPlan] runtimeConfig.inference.session.decodeLoop is required.');
211
+ }
162
212
 
163
213
  const finitenessPolicy = resolveRangeAwareSelectiveWideningConfig(computeConfig);
164
214
  const deferredRoundingWindowTokens = resolveDeferredRoundingWindowTokens(computeConfig);
215
+ const batchingConfig = {
216
+ batchSize: decodeLoopConfig.batchSize,
217
+ stopCheckMode: decodeLoopConfig.stopCheckMode,
218
+ maxTokens: generationConfig.maxTokens ?? DEFAULT_MAX_TOKENS,
219
+ readbackInterval: decodeLoopConfig.readbackInterval,
220
+ readbackMode: assertReadbackMode(decodeLoopConfig.readbackMode),
221
+ ringTokens: decodeLoopConfig.ringTokens,
222
+ ringStop: decodeLoopConfig.ringStop,
223
+ ringStaging: decodeLoopConfig.ringStaging,
224
+ disableCommandBatching: assertOptionalBoolean(
225
+ decodeLoopConfig.disableCommandBatching,
226
+ 'runtimeConfig.inference.session.decodeLoop.disableCommandBatching'
227
+ ),
228
+ };
165
229
 
166
230
  const primaryPlan = createStaticExecutionPlan({
167
231
  id: PRIMARY_EXECUTION_PLAN_ID,
@@ -174,9 +238,11 @@ export function compileExecutionPlanState(options) {
174
238
  generationConfig,
175
239
  batchingConfig,
176
240
  });
241
+ const primaryUsesF16ActivationExecution = primaryPlan.activationDtype === 'f16'
242
+ || kernelPathUsesF16ActivationExecution(resolvedKernelPath);
177
243
 
178
244
  let fallbackPlan = null;
179
- if (primaryPlan.finitenessGuardEnabled) {
245
+ if (primaryUsesF16ActivationExecution && primaryPlan.finitenessOnTrigger === 'fallback-plan') {
180
246
  const fallbackActivationDtype = resolveFallbackActivationDtype(primaryPlan.activationDtype);
181
247
  if (fallbackActivationDtype !== 'f32') {
182
248
  throw new Error(
@@ -305,6 +371,10 @@ function resolveExecutionOverrides(options = {}) {
305
371
  batchSize: assertOptionalPositiveInt(options.batchSize, 'batchSize'),
306
372
  stopCheckMode: assertOptionalStopCheckMode(options.stopCheckMode),
307
373
  maxTokens: assertOptionalPositiveInt(options.maxTokens, 'maxTokens'),
374
+ readbackInterval: assertOptionalPositiveInt(options.readbackInterval, 'readbackInterval'),
375
+ ringTokens: assertOptionalPositiveInt(options.ringTokens, 'ringTokens'),
376
+ ringStop: assertOptionalPositiveInt(options.ringStop, 'ringStop'),
377
+ ringStaging: assertOptionalPositiveInt(options.ringStaging, 'ringStaging'),
308
378
  };
309
379
  }
310
380
 
@@ -319,6 +389,7 @@ export function resolveExecutionSessionPlan(container, options = {}) {
319
389
  kernelPathId: activePlan.kernelPathId,
320
390
  activationDtype: activePlan.activationDtype,
321
391
  finitenessGuardEnabled: activePlan.finitenessGuardEnabled,
392
+ finitenessOnTrigger: activePlan.finitenessOnTrigger,
322
393
  finitenessAbsThreshold: activePlan.finitenessAbsThreshold,
323
394
  finitenessIncludeNonFinite: activePlan.finitenessIncludeNonFinite,
324
395
  deferredRoundingWindowTokens: activePlan.deferredRoundingWindowTokens,
@@ -327,10 +398,11 @@ export function resolveExecutionSessionPlan(container, options = {}) {
327
398
  batchSize: overrides.batchSize ?? activePlan.defaultBatchSize,
328
399
  stopCheckMode: overrides.stopCheckMode ?? activePlan.defaultStopCheckMode,
329
400
  maxTokens: overrides.maxTokens ?? activePlan.defaultMaxTokens,
330
- readbackInterval: activePlan.readbackInterval,
331
- ringTokens: activePlan.ringTokens,
332
- ringStop: activePlan.ringStop,
333
- ringStaging: activePlan.ringStaging,
401
+ readbackInterval: overrides.readbackInterval ?? activePlan.readbackInterval,
402
+ readbackMode: activePlan.readbackMode,
403
+ ringTokens: overrides.ringTokens ?? activePlan.ringTokens,
404
+ ringStop: overrides.ringStop ?? activePlan.ringStop,
405
+ ringStaging: overrides.ringStaging ?? activePlan.ringStaging,
334
406
  overrides,
335
407
  };
336
408
  }
@@ -349,9 +421,29 @@ export function isBatchDecodeEnabled(config) {
349
421
  disableCommandBatching: config.disableCommandBatching,
350
422
  isBdpaPagedLayout: config.isBdpaPagedLayout === true,
351
423
  finitenessFallbackWindowOpen: config.finitenessFallbackWindowOpen === true,
424
+ hasLinearAttentionLayers: config.hasLinearAttentionLayers === true,
425
+ selfSpeculationEnabled: config.selfSpeculationEnabled === true,
426
+ hasRangeBackedPerLayerInputs: config.hasRangeBackedPerLayerInputs === true,
352
427
  });
353
428
  }
354
429
 
430
+ export function resolveMaxBatchDecodeTokens(config) {
431
+ const value = selectRuleValue('inference', 'execution', 'maxBatchDecodeTokens', {
432
+ hasHotVocabularyBatchDecode: config.hasHotVocabularyBatchDecode === true,
433
+ hasGpuSplitPerLayerInputs: config.hasGpuSplitPerLayerInputs === true,
434
+ hasLinearAttentionLayers: config.hasLinearAttentionLayers === true,
435
+ });
436
+ if (value == null) {
437
+ return null;
438
+ }
439
+ if (!Number.isInteger(value) || value < 1) {
440
+ throw new Error(
441
+ `[ExecutionPlan] maxBatchDecodeTokens must be null or a positive integer; got ${JSON.stringify(value)}.`
442
+ );
443
+ }
444
+ return value;
445
+ }
446
+
355
447
  export function isDecodeRecorderEnabled(config) {
356
448
  return selectRuleValue('inference', 'execution', 'decodeRecorderEnabled', {
357
449
  hasDevice: config.hasDevice === true,
@@ -4,7 +4,7 @@ export declare function normalizeDtype(value: unknown, label: string): 'f16' | '
4
4
  export declare function isPhaseMatch(phase: string, targetPhase: string): boolean;
5
5
  export declare function stepHasLayer(step: { layers: 'all' | number[] }, layerIdx: number): boolean;
6
6
  export declare function requireSessionActivationDtype(
7
- sessionDefaults: Record<string, unknown> | null | undefined,
7
+ session: Record<string, unknown> | null | undefined,
8
8
  label?: string
9
9
  ): 'f16' | 'f32';
10
10
 
@@ -14,7 +14,7 @@ export declare function resolveFinitenessFallbackKernelPathId(
14
14
 
15
15
  export declare function buildInlineKernelPath(
16
16
  steps: readonly Record<string, unknown>[],
17
- sessionDefaults: Record<string, unknown> | null,
17
+ session: Record<string, unknown> | null,
18
18
  modelId: string,
19
19
  numLayers: number,
20
20
  finitenessFallbackKernelPathId?: string | null
@@ -22,12 +22,16 @@ export declare function buildInlineKernelPath(
22
22
 
23
23
  export declare function assertKernelPathSessionCompatibility(
24
24
  path: Record<string, unknown> | null | undefined,
25
- sessionDefaults: Record<string, unknown> | null | undefined
25
+ session: Record<string, unknown> | null | undefined
26
26
  ): void;
27
27
 
28
28
  export interface BuildLayerPipelineOptions {
29
29
  /** When true, throws on incompatible ops instead of returning a degraded result. */
30
30
  strict?: boolean;
31
+ /** When false, suppresses the degraded-pipeline log because another execution path remains active. */
32
+ logIncompatibleOps?: boolean;
33
+ /** When set, fused `ffn` steps inherit this dtype when they do not declare per-step precision. */
34
+ ffnDtypeFallback?: 'f16' | 'f32' | null;
31
35
  }
32
36
 
33
37
  export declare function buildLayerPipelineFromExecution(
@@ -38,7 +42,7 @@ export declare function buildLayerPipelineFromExecution(
38
42
  | null;
39
43
 
40
44
  export declare function buildSessionRuntimePatch(
41
- sessionDefaults: Record<string, unknown> | null | undefined,
45
+ session: Record<string, unknown> | null | undefined,
42
46
  options?: {
43
47
  includeDecodeLoop?: boolean;
44
48
  }
@@ -36,10 +36,10 @@ export function stepHasLayer(step, layerIdx) {
36
36
  }
37
37
 
38
38
  export function requireSessionActivationDtype(
39
- sessionDefaults,
40
- label = 'sessionDefaults.compute.defaults.activationDtype'
39
+ session,
40
+ label = 'session.compute.defaults.activationDtype'
41
41
  ) {
42
- const activationDtype = sessionDefaults?.compute?.defaults?.activationDtype;
42
+ const activationDtype = session?.compute?.defaults?.activationDtype;
43
43
  if (activationDtype == null) {
44
44
  throw new Error(`[Execution] ${label} is required.`);
45
45
  }
@@ -66,6 +66,22 @@ function toKernelPathStep(step) {
66
66
  };
67
67
  }
68
68
 
69
+ function applyFusedFfnKernelPathPrecision(step, activationDtype) {
70
+ if (!step || step.op !== 'ffn') {
71
+ return step;
72
+ }
73
+ const inputDtype = step.precision?.inputDtype ?? activationDtype;
74
+ const outputDtype = step.precision?.outputDtype ?? activationDtype;
75
+ return {
76
+ ...step,
77
+ precision: {
78
+ ...(step.precision ?? {}),
79
+ inputDtype,
80
+ outputDtype,
81
+ },
82
+ };
83
+ }
84
+
69
85
  function getSectionSteps(steps, section, phase = null) {
70
86
  return steps
71
87
  .filter((step) => step.section === section)
@@ -89,20 +105,24 @@ function getInlineKernelPathSteps(path) {
89
105
  ...(path?.prefill?.steps ?? []),
90
106
  ...(path?.postLayer ?? []),
91
107
  ...(path?.sampling ?? []),
92
- ...(path?.layerOverrides?.flatMap((override) => override.steps ?? []) ?? []),
108
+ ...(path?.layerOverrides?.flatMap((override) => [
109
+ ...(override?.steps ?? []),
110
+ ...(override?.decode?.steps ?? []),
111
+ ...(override?.prefill?.steps ?? []),
112
+ ]) ?? []),
93
113
  ];
94
114
  }
95
115
 
96
- export function assertKernelPathSessionCompatibility(path, sessionDefaults) {
116
+ export function assertKernelPathSessionCompatibility(path, session) {
97
117
  if (!path) {
98
118
  return;
99
119
  }
100
120
  const globalActivationDtype = normalizeDtype(
101
- path.activationDtype ?? requireSessionActivationDtype(sessionDefaults),
121
+ path.activationDtype ?? requireSessionActivationDtype(session),
102
122
  'inlineKernelPath.activationDtype'
103
123
  );
104
124
  const kvDtype = normalizeDtype(
105
- path.kvDtype ?? sessionDefaults?.kvcache?.kvDtype ?? globalActivationDtype,
125
+ path.kvDtype ?? session?.kvcache?.kvDtype ?? globalActivationDtype,
106
126
  'inlineKernelPath.kvDtype'
107
127
  );
108
128
 
@@ -161,15 +181,15 @@ export function resolveFinitenessFallbackKernelPathId(kernelPathId) {
161
181
 
162
182
  export function buildInlineKernelPath(
163
183
  steps,
164
- sessionDefaults,
184
+ session,
165
185
  modelId,
166
186
  numLayers,
167
187
  finitenessFallbackKernelPathId = null
168
188
  ) {
169
- const activationDtype = requireSessionActivationDtype(sessionDefaults);
189
+ const activationDtype = requireSessionActivationDtype(session);
170
190
  const kvDtype = normalizeDtype(
171
- sessionDefaults?.kvcache?.kvDtype ?? activationDtype,
172
- 'sessionDefaults.kvcache.kvDtype'
191
+ session?.kvcache?.kvDtype ?? activationDtype,
192
+ 'session.kvcache.kvDtype'
173
193
  );
174
194
  const decodeSteps = buildLayerPhaseSteps(steps, 'decode', 0);
175
195
  const prefillSteps = buildLayerPhaseSteps(steps, 'prefill', 0);
@@ -193,24 +213,35 @@ export function buildInlineKernelPath(
193
213
  steps: prefillSteps.length > 0 ? prefillSteps : decodeSteps,
194
214
  },
195
215
  };
216
+ path.decode.steps = path.decode.steps.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
217
+ path.prefill.steps = path.prefill.steps.map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
196
218
 
197
219
  if (numLayers > 0) {
198
220
  const overrides = [];
199
221
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
200
- const decodeLayerSteps = buildLayerPhaseSteps(steps, 'decode', layerIdx);
201
- const prefillLayerSteps = buildLayerPhaseSteps(steps, 'prefill', layerIdx);
222
+ const decodeLayerSteps = buildLayerPhaseSteps(steps, 'decode', layerIdx)
223
+ .map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
224
+ const prefillLayerSteps = buildLayerPhaseSteps(steps, 'prefill', layerIdx)
225
+ .map((step) => applyFusedFfnKernelPathPrecision(step, activationDtype));
202
226
  const hasCustomDecode = JSON.stringify(decodeLayerSteps) !== JSON.stringify(path.decode.steps);
203
227
  const hasCustomPrefill = JSON.stringify(prefillLayerSteps) !== JSON.stringify(path.prefill.steps);
204
228
  if (!hasCustomDecode && !hasCustomPrefill) continue;
205
- const mergedLayerSteps = decodeLayerSteps.length > 0
206
- ? decodeLayerSteps
207
- : prefillLayerSteps;
208
- if (mergedLayerSteps.length > 0) {
209
- overrides.push({
210
- layers: [layerIdx],
211
- steps: mergedLayerSteps,
212
- });
229
+ const override = { layers: [layerIdx] };
230
+ if (
231
+ hasCustomDecode
232
+ && hasCustomPrefill
233
+ && JSON.stringify(decodeLayerSteps) === JSON.stringify(prefillLayerSteps)
234
+ ) {
235
+ override.steps = decodeLayerSteps;
236
+ } else {
237
+ if (hasCustomDecode && decodeLayerSteps.length > 0) {
238
+ override.decode = { steps: decodeLayerSteps };
239
+ }
240
+ if (hasCustomPrefill && prefillLayerSteps.length > 0) {
241
+ override.prefill = { steps: prefillLayerSteps };
242
+ }
213
243
  }
244
+ overrides.push(override);
214
245
  }
215
246
  if (overrides.length > 0) {
216
247
  path.layerOverrides = overrides;
@@ -230,7 +261,7 @@ export function buildInlineKernelPath(
230
261
  path.sampling = sampling;
231
262
  }
232
263
 
233
- assertKernelPathSessionCompatibility(path, sessionDefaults);
264
+ assertKernelPathSessionCompatibility(path, session);
234
265
  return path;
235
266
  }
236
267
 
@@ -244,7 +275,10 @@ export function buildInlineKernelPath(
244
275
  * | { incompatibleOps: string[]; hasIncompatibleOps: true } | null}
245
276
  */
246
277
  export function buildLayerPipelineFromExecution(steps, options = {}) {
247
- const { strict = false } = options;
278
+ const { strict = false, logIncompatibleOps = true } = options;
279
+ const ffnDtypeFallback = options.ffnDtypeFallback == null
280
+ ? null
281
+ : normalizeDtype(options.ffnDtypeFallback, 'buildLayerPipelineFromExecution.options.ffnDtypeFallback');
248
282
  const layerSectionSteps = steps.filter((step) => step.section === 'layer');
249
283
  if (layerSectionSteps.length === 0) {
250
284
  return null;
@@ -257,13 +291,21 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
257
291
  ),
258
292
  ];
259
293
  if (incompatibleOps.length > 0) {
260
- const message =
294
+ const degradedMessage =
261
295
  `[Execution] Layer pipeline contains ops not in PIPELINE_COMPATIBLE_OPS: ` +
262
296
  `${incompatibleOps.join(', ')}. Pipeline will be degraded.`;
263
297
  if (strict) {
264
- throw new Error(message);
298
+ throw new Error(degradedMessage);
299
+ }
300
+ if (logIncompatibleOps) {
301
+ log.error('ExecutionRuntime', degradedMessage);
302
+ } else {
303
+ log.debug(
304
+ 'ExecutionRuntime',
305
+ `[Execution] Layer pipeline contains ops not in PIPELINE_COMPATIBLE_OPS: ` +
306
+ `${incompatibleOps.join(', ')}. Inline kernel path remains active; JS layer pipeline disabled.`
307
+ );
265
308
  }
266
- log.error('ExecutionRuntime', message);
267
309
  return { incompatibleOps, hasIncompatibleOps: true };
268
310
  }
269
311
 
@@ -278,8 +320,13 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
278
320
  ...(step.b !== undefined ? { b: step.b } : {}),
279
321
  ...(step.variant !== undefined ? { variant: step.variant } : {}),
280
322
  ...(step.skipInputNorm !== undefined ? { skipInputNorm: step.skipInputNorm } : {}),
281
- ...(step.precision?.inputDtype ? { inputDtype: step.precision.inputDtype } : {}),
282
- ...(step.precision?.outputDtype ? { outputDtype: step.precision.outputDtype } : {}),
323
+ ...(step.precision?.inputDtype
324
+ ? { inputDtype: step.precision.inputDtype }
325
+ : (step.op === 'ffn' && ffnDtypeFallback ? { inputDtype: ffnDtypeFallback } : {})),
326
+ ...(step.precision?.outputDtype
327
+ ? { outputDtype: step.precision.outputDtype }
328
+ : (step.op === 'ffn' && ffnDtypeFallback ? { outputDtype: ffnDtypeFallback } : {})),
329
+ ...(step.precision?.kvDtype ? { kvDtype: step.precision.kvDtype } : {}),
283
330
  ...(step.fromDtype ? { fromDtype: step.fromDtype } : {}),
284
331
  ...(step.toDtype ? { toDtype: step.toDtype } : {}),
285
332
  ...(step.probeStage ? { probeStage: step.probeStage } : {}),
@@ -295,7 +342,7 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
295
342
  }
296
343
 
297
344
  /**
298
- * Build a runtime config patch from manifest sessionDefaults.
345
+ * Build a runtime config patch from manifest/runtime session.
299
346
  *
300
347
  * Field consumption status after merge into runtimeConfig.inference:
301
348
  *
@@ -304,10 +351,9 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
304
351
  * logits fallback (getRuntimeConfig().inference.compute.activationDtype),
305
352
  * and layer context builder.
306
353
  *
307
- * CONSUMED (read by KV cache and batching subsystems):
308
- * - patch.kvcache.*
354
+ * CONSUMED (read by KV cache, batching, and execution-plan subsystems):
355
+ * - patch.session.kvcache.*
309
356
  * - patch.batching.*
310
- * - patch.generation.disableCommandBatching
311
357
  *
312
358
  * DEAD / NOT CONSUMED at runtime (merged into runtimeConfig but never read back):
313
359
  * - patch.session.compute.defaults.mathDtype
@@ -318,10 +364,10 @@ export function buildLayerPipelineFromExecution(steps, options = {}) {
318
364
  * future consumption. They should NOT be removed (non-breaking), but new code
319
365
  * should not rely on reading them from runtimeConfig.inference.session.
320
366
  */
321
- export function buildSessionRuntimePatch(sessionDefaults, options = {}) {
367
+ export function buildSessionRuntimePatch(session, options = {}) {
322
368
  const includeDecodeLoop = options.includeDecodeLoop !== false;
323
369
  const patch = {};
324
- const computeDefaults = sessionDefaults?.compute?.defaults ?? null;
370
+ const computeDefaults = session?.compute?.defaults ?? null;
325
371
  const computePatch = {};
326
372
  const sessionComputeDefaultsPatch = {};
327
373
  const activationDtype = computeDefaults?.activationDtype;
@@ -360,23 +406,18 @@ export function buildSessionRuntimePatch(sessionDefaults, options = {}) {
360
406
  },
361
407
  };
362
408
  }
363
- if (sessionDefaults?.kvcache) {
364
- patch.kvcache = sessionDefaults.kvcache;
365
- }
366
- if (includeDecodeLoop && sessionDefaults?.decodeLoop) {
409
+ if (includeDecodeLoop && session?.decodeLoop) {
367
410
  patch.batching = {
368
- batchSize: sessionDefaults.decodeLoop.batchSize,
369
- stopCheckMode: sessionDefaults.decodeLoop.stopCheckMode,
370
- readbackInterval: sessionDefaults.decodeLoop.readbackInterval,
371
- ringTokens: sessionDefaults.decodeLoop.ringTokens,
372
- ringStop: sessionDefaults.decodeLoop.ringStop,
373
- ringStaging: sessionDefaults.decodeLoop.ringStaging,
411
+ batchSize: session.decodeLoop.batchSize,
412
+ stopCheckMode: session.decodeLoop.stopCheckMode,
413
+ readbackInterval: session.decodeLoop.readbackInterval,
414
+ ringTokens: session.decodeLoop.ringTokens,
415
+ ringStop: session.decodeLoop.ringStop,
416
+ ringStaging: session.decodeLoop.ringStaging,
374
417
  };
375
- if (sessionDefaults.decodeLoop.disableCommandBatching !== undefined) {
376
- patch.generation = {
377
- disableCommandBatching: sessionDefaults.decodeLoop.disableCommandBatching === true,
378
- };
379
- }
418
+ }
419
+ if (session) {
420
+ patch.session = session;
380
421
  }
381
422
  return patch;
382
423
  }
@@ -1,12 +1,13 @@
1
1
  import type {
2
2
  ExecutionV1GraphSchema,
3
- ExecutionV1SessionDefaultsSchema,
3
+ ExecutionV1SessionSchema,
4
4
  ExecutionV1PoliciesSchema,
5
5
  ExecutionV1ExpandedStepSchema,
6
6
  } from '../../../config/schema/execution-v1.schema.js';
7
+ import type { KernelPathSchema } from '../../../config/schema/index.js';
7
8
 
8
9
  export interface ExecutionV1CompiledState {
9
- sessionDefaults: ExecutionV1SessionDefaultsSchema;
10
+ session: ExecutionV1SessionSchema;
10
11
  policies: ExecutionV1PoliciesSchema;
11
12
  resolvedSteps: {
12
13
  prefill: ExecutionV1ExpandedStepSchema[];
@@ -14,6 +15,8 @@ export interface ExecutionV1CompiledState {
14
15
  all: ExecutionV1ExpandedStepSchema[];
15
16
  };
16
17
  runtimeInferencePatch: Record<string, unknown>;
18
+ appliedTransforms?: string[];
19
+ fallbackKernelPath?: KernelPathSchema | null;
17
20
  }
18
21
 
19
22
  export declare function hasExecutionV1(
@@ -24,16 +27,32 @@ export declare function compileExecutionV1(options?: {
24
27
  manifestInference: {
25
28
  schema: string;
26
29
  execution: ExecutionV1GraphSchema;
27
- sessionDefaults: ExecutionV1SessionDefaultsSchema;
30
+ session: ExecutionV1SessionSchema;
28
31
  };
29
32
  modelId?: string;
30
33
  numLayers?: number;
34
+ runtimeSession?: ExecutionV1SessionSchema | null;
35
+ runtimeCompute?: Record<string, unknown> | null;
36
+ kernelPathPolicy?: Record<string, unknown> | null;
37
+ capabilities?: {
38
+ hasSubgroups?: boolean;
39
+ hasF16?: boolean;
40
+ hasSubgroupsF16?: boolean;
41
+ maxWorkgroupSize?: number;
42
+ maxBufferSize?: number;
43
+ } | null;
44
+ platform?: {
45
+ id?: string;
46
+ vendor?: string;
47
+ architecture?: string;
48
+ } | null;
31
49
  }): ExecutionV1CompiledState;
32
50
 
33
51
  export declare function applyExecutionV1RuntimeConfig(options?: {
34
52
  runtimeConfig: Record<string, unknown>;
53
+ runtimeOverrides?: Record<string, unknown> | null;
35
54
  manifest: {
36
- inference?: { schema?: string; execution?: ExecutionV1GraphSchema; sessionDefaults?: ExecutionV1SessionDefaultsSchema };
55
+ inference?: { schema?: string; execution?: ExecutionV1GraphSchema; session?: ExecutionV1SessionSchema };
37
56
  modelId?: string;
38
57
  architecture?: { numLayers?: number };
39
58
  };