doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -14,14 +14,7 @@ import { KERNEL_CONFIGS } from '../../../gpu/kernels/kernel-configs.js';
14
14
  import { initTokenizer } from './init.js';
15
15
  import { selectRuleValue } from '../../../rules/rule-registry.js';
16
16
  import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
17
- import {
18
- DEFAULT_BATCHING_DEFAULTS,
19
- DEFAULT_COMPUTE_DEFAULTS,
20
- DEFAULT_GENERATION_CONFIG,
21
- DEFAULT_KERNEL_PATH_POLICY as SCHEMA_DEFAULT_KERNEL_PATH_POLICY,
22
- } from '../../../config/schema/inference-defaults.schema.js';
23
- import { DEFAULT_KVCACHE_CONFIG } from '../../../config/schema/kvcache.schema.js';
24
- import { DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS } from '../../../config/schema/execution-v1.schema.js';
17
+ import { READBACK_MODES } from '../../../config/schema/execution-v1.schema.js';
25
18
 
26
19
  function validateKernelWarmupMode(mode) {
27
20
  if (mode !== 'parallel' && mode !== 'sequential') {
@@ -51,7 +44,7 @@ function normalizeBoolean(value) {
51
44
  return typeof value === 'boolean' ? value : null;
52
45
  }
53
46
 
54
- function parseManifestDecodeLoopOptionalPositiveInt(value, label, modelId) {
47
+ function parseSessionDecodeLoopOptionalPositiveInt(value, label, modelId) {
55
48
  if (value === undefined) {
56
49
  return undefined;
57
50
  }
@@ -61,144 +54,110 @@ function parseManifestDecodeLoopOptionalPositiveInt(value, label, modelId) {
61
54
  const normalized = normalizePositiveInt(value);
62
55
  if (normalized == null) {
63
56
  throw new Error(
64
- `Manifest "${modelId}" inference.sessionDefaults.decodeLoop.${label} must be a positive integer or null.`
57
+ `Manifest "${modelId}" inference.session.decodeLoop.${label} must be a positive integer or null.`
65
58
  );
66
59
  }
67
60
  return normalized;
68
61
  }
69
62
 
70
- function parseManifestDecodeLoopOptionalBoolean(value, label, modelId) {
63
+ function parseSessionDecodeLoopOptionalBoolean(value, label, modelId) {
71
64
  if (value === undefined) {
72
65
  return undefined;
73
66
  }
74
67
  if (typeof value !== 'boolean') {
75
68
  throw new Error(
76
- `Manifest "${modelId}" inference.sessionDefaults.decodeLoop.${label} must be a boolean when provided.`
69
+ `Manifest "${modelId}" inference.session.decodeLoop.${label} must be a boolean when provided.`
77
70
  );
78
71
  }
79
72
  return value;
80
73
  }
81
74
 
82
- function requireGlobalBatchingDefault(value, label) {
75
+ function requireSessionDecodeLoopPositiveInt(value, label, modelId) {
83
76
  const normalized = normalizePositiveInt(value);
84
77
  if (normalized == null) {
85
- throw new Error(`${label} must be a positive integer.`);
78
+ throw new Error(`Manifest "${modelId}" inference.session.decodeLoop.${label} must be a positive integer.`);
86
79
  }
87
80
  return normalized;
88
81
  }
89
82
 
90
- function requireGlobalStopCheckMode(value, label) {
83
+ function requireSessionDecodeLoopStopCheckMode(value, modelId) {
91
84
  const normalized = normalizeStopCheckMode(value);
92
85
  if (normalized == null) {
93
- throw new Error(`${label} must be "batch" or "per-token".`);
86
+ throw new Error(
87
+ `Manifest "${modelId}" inference.session.decodeLoop.stopCheckMode must be "batch" or "per-token".`
88
+ );
94
89
  }
95
90
  return normalized;
96
91
  }
97
92
 
98
- const GLOBAL_DEFAULT_BATCHING = Object.freeze({
99
- batchSize: requireGlobalBatchingDefault(
100
- DEFAULT_BATCHING_DEFAULTS.batchSize,
101
- 'DEFAULT_BATCHING_DEFAULTS.batchSize'
102
- ),
103
- stopCheckMode: requireGlobalStopCheckMode(
104
- DEFAULT_BATCHING_DEFAULTS.stopCheckMode,
105
- 'DEFAULT_BATCHING_DEFAULTS.stopCheckMode'
106
- ),
107
- readbackInterval: requireGlobalBatchingDefault(
108
- DEFAULT_BATCHING_DEFAULTS.readbackInterval,
109
- 'DEFAULT_BATCHING_DEFAULTS.readbackInterval'
110
- ),
111
- ringTokens: requireGlobalBatchingDefault(
112
- DEFAULT_BATCHING_DEFAULTS.ringTokens,
113
- 'DEFAULT_BATCHING_DEFAULTS.ringTokens'
114
- ),
115
- ringStop: requireGlobalBatchingDefault(
116
- DEFAULT_BATCHING_DEFAULTS.ringStop,
117
- 'DEFAULT_BATCHING_DEFAULTS.ringStop'
118
- ),
119
- ringStaging: requireGlobalBatchingDefault(
120
- DEFAULT_BATCHING_DEFAULTS.ringStaging,
121
- 'DEFAULT_BATCHING_DEFAULTS.ringStaging'
122
- ),
123
- });
124
-
125
- const GLOBAL_DEFAULT_GENERATION = Object.freeze({
126
- disableCommandBatching: DEFAULT_GENERATION_CONFIG.disableCommandBatching === true,
127
- });
128
-
129
- const GLOBAL_DEFAULT_KERNEL_PATH_DTYPES = Object.freeze({
130
- activationDtype: DEFAULT_COMPUTE_DEFAULTS.activationDtype,
131
- kvDtype: DEFAULT_KVCACHE_CONFIG.kvDtype,
132
- outputDtype: DEFAULT_EXECUTION_V1_COMPUTE_DEFAULTS.outputDtype,
133
- });
134
-
135
- function isRuntimeBatchingAtGlobalDefaults(batching) {
136
- if (!batching || typeof batching !== 'object') {
137
- return false;
138
- }
139
- return normalizePositiveInt(batching.batchSize) === GLOBAL_DEFAULT_BATCHING.batchSize
140
- && normalizeStopCheckMode(batching.stopCheckMode) === GLOBAL_DEFAULT_BATCHING.stopCheckMode
141
- && normalizeReadbackInterval(batching.readbackInterval) === GLOBAL_DEFAULT_BATCHING.readbackInterval
142
- && normalizeReadbackInterval(batching.ringTokens) === GLOBAL_DEFAULT_BATCHING.ringTokens
143
- && normalizeReadbackInterval(batching.ringStop) === GLOBAL_DEFAULT_BATCHING.ringStop
144
- && normalizeReadbackInterval(batching.ringStaging) === GLOBAL_DEFAULT_BATCHING.ringStaging;
145
- }
146
-
147
- function isRuntimeGenerationAtGlobalDefaults(generation) {
148
- if (!generation || typeof generation !== 'object') {
149
- return false;
93
+ function resolveDecodeLoopRuntimeSession(runtimeConfig, runtimeOverrides, useExplicitRuntimeOverrides) {
94
+ if (!useExplicitRuntimeOverrides) {
95
+ return runtimeConfig?.inference?.session ?? null;
150
96
  }
151
- return (generation.disableCommandBatching === true) === GLOBAL_DEFAULT_GENERATION.disableCommandBatching;
152
- }
153
-
154
- function requireManifestDecodeLoopPositiveInt(value, label, modelId) {
155
- const normalized = normalizePositiveInt(value);
156
- if (normalized == null) {
157
- throw new Error(`Manifest "${modelId}" inference.sessionDefaults.decodeLoop.${label} must be a positive integer.`);
97
+ const inferenceOverrides = runtimeOverrides?.inference;
98
+ if (!inferenceOverrides || typeof inferenceOverrides !== 'object' || Array.isArray(inferenceOverrides)) {
99
+ return null;
158
100
  }
159
- return normalized;
160
- }
161
-
162
- function requireManifestDecodeLoopStopCheckMode(value, modelId) {
163
- const normalized = normalizeStopCheckMode(value);
164
- if (normalized == null) {
165
- throw new Error(
166
- `Manifest "${modelId}" inference.sessionDefaults.decodeLoop.stopCheckMode must be "batch" or "per-token".`
167
- );
101
+ if (!Object.prototype.hasOwnProperty.call(inferenceOverrides, 'session')) {
102
+ return null;
168
103
  }
169
- return normalized;
104
+ return inferenceOverrides.session ?? null;
170
105
  }
171
106
 
172
- function buildManifestDecodeLoopRuntimePatch(manifest) {
173
- const decodeLoop = manifest?.inference?.sessionDefaults?.decodeLoop;
107
+ function buildResolvedDecodeLoopRuntimePatch(
108
+ runtimeConfig,
109
+ manifest,
110
+ runtimeOverrides = undefined,
111
+ useExplicitRuntimeOverrides = false
112
+ ) {
113
+ const runtimeSession = resolveDecodeLoopRuntimeSession(
114
+ runtimeConfig,
115
+ runtimeOverrides,
116
+ useExplicitRuntimeOverrides
117
+ );
118
+ const resolvedSession = mergeRuntimeValues(
119
+ manifest?.inference?.session ?? {},
120
+ runtimeSession ?? {}
121
+ );
122
+ const decodeLoop = resolvedSession?.decodeLoop;
174
123
  if (decodeLoop == null) {
175
124
  return null;
176
125
  }
177
126
  const modelId = String(manifest?.modelId ?? 'unknown').trim() || 'unknown';
178
127
  if (typeof decodeLoop !== 'object') {
179
128
  throw new Error(
180
- `Manifest "${modelId}" inference.sessionDefaults.decodeLoop must be an object when provided.`
129
+ `Manifest "${modelId}" inference.session.decodeLoop must be an object when provided.`
181
130
  );
182
131
  }
183
- const batchSize = requireManifestDecodeLoopPositiveInt(decodeLoop.batchSize, 'batchSize', modelId);
184
- const stopCheckMode = requireManifestDecodeLoopStopCheckMode(decodeLoop.stopCheckMode, modelId);
185
- const readbackInterval = requireManifestDecodeLoopPositiveInt(
132
+ const batchSize = requireSessionDecodeLoopPositiveInt(decodeLoop.batchSize, 'batchSize', modelId);
133
+ const stopCheckMode = requireSessionDecodeLoopStopCheckMode(decodeLoop.stopCheckMode, modelId);
134
+ const readbackInterval = requireSessionDecodeLoopPositiveInt(
186
135
  decodeLoop.readbackInterval,
187
136
  'readbackInterval',
188
137
  modelId
189
138
  );
190
- const disableCommandBatching = parseManifestDecodeLoopOptionalBoolean(
139
+ const disableCommandBatching = parseSessionDecodeLoopOptionalBoolean(
191
140
  decodeLoop.disableCommandBatching,
192
141
  'disableCommandBatching',
193
142
  modelId
194
143
  );
195
144
 
145
+ const readbackMode = decodeLoop.readbackMode;
146
+ if (!readbackMode || !READBACK_MODES.includes(readbackMode)) {
147
+ throw new Error(
148
+ `DopplerConfigError: Manifest "${modelId}" inference.session.decodeLoop.readbackMode ` +
149
+ `is required and must be one of ${READBACK_MODES.join(', ')}; got ${JSON.stringify(readbackMode)}. ` +
150
+ 'Set it explicitly in the manifest session.decodeLoop.'
151
+ );
152
+ }
153
+ const submitLatencyThresholdMs = decodeLoop.submitLatencyThresholdMs ?? null;
154
+
196
155
  const batchingPatch = {
197
156
  batchSize,
198
157
  stopCheckMode,
199
158
  readbackInterval,
200
159
  };
201
- const ringTokens = parseManifestDecodeLoopOptionalPositiveInt(
160
+ const ringTokens = parseSessionDecodeLoopOptionalPositiveInt(
202
161
  decodeLoop.ringTokens,
203
162
  'ringTokens',
204
163
  modelId
@@ -206,7 +165,7 @@ function buildManifestDecodeLoopRuntimePatch(manifest) {
206
165
  if (ringTokens !== undefined) {
207
166
  batchingPatch.ringTokens = ringTokens;
208
167
  }
209
- const ringStop = parseManifestDecodeLoopOptionalPositiveInt(
168
+ const ringStop = parseSessionDecodeLoopOptionalPositiveInt(
210
169
  decodeLoop.ringStop,
211
170
  'ringStop',
212
171
  modelId
@@ -214,7 +173,7 @@ function buildManifestDecodeLoopRuntimePatch(manifest) {
214
173
  if (ringStop !== undefined) {
215
174
  batchingPatch.ringStop = ringStop;
216
175
  }
217
- const ringStaging = parseManifestDecodeLoopOptionalPositiveInt(
176
+ const ringStaging = parseSessionDecodeLoopOptionalPositiveInt(
218
177
  decodeLoop.ringStaging,
219
178
  'ringStaging',
220
179
  modelId
@@ -224,54 +183,101 @@ function buildManifestDecodeLoopRuntimePatch(manifest) {
224
183
  }
225
184
 
226
185
  return {
186
+ session: {
187
+ ...resolvedSession,
188
+ decodeLoop: {
189
+ ...decodeLoop,
190
+ batchSize,
191
+ stopCheckMode,
192
+ readbackInterval,
193
+ readbackMode,
194
+ submitLatencyThresholdMs,
195
+ ...(ringTokens !== undefined ? { ringTokens } : {}),
196
+ ...(ringStop !== undefined ? { ringStop } : {}),
197
+ ...(ringStaging !== undefined ? { ringStaging } : {}),
198
+ ...(disableCommandBatching !== undefined ? { disableCommandBatching } : {}),
199
+ },
200
+ },
227
201
  batching: batchingPatch,
228
- generation: disableCommandBatching == null
229
- ? null
230
- : { disableCommandBatching: disableCommandBatching === true },
231
202
  };
232
203
  }
233
204
 
234
- export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, modelConfig) {
205
+ export function applyModelBatchingRuntimeDefaults(runtimeConfig, manifest, modelConfig, runtimeOverrides = undefined) {
235
206
  void modelConfig;
236
- const batching = runtimeConfig?.inference?.batching;
237
- const generation = runtimeConfig?.inference?.generation;
238
- const runtimeBatchingAtDefaults = isRuntimeBatchingAtGlobalDefaults(batching);
239
- const runtimeGenerationAtDefaults = isRuntimeGenerationAtGlobalDefaults(generation);
240
-
241
- const patch = buildManifestDecodeLoopRuntimePatch(manifest);
207
+ const patch = buildResolvedDecodeLoopRuntimePatch(
208
+ runtimeConfig,
209
+ manifest,
210
+ runtimeOverrides,
211
+ arguments.length > 3
212
+ );
242
213
  if (!patch) {
243
214
  return runtimeConfig;
244
215
  }
245
216
 
246
- const runtimeDisableCommandBatching = generation?.disableCommandBatching === true;
247
- const manifestDisableCommandBatching = patch.generation?.disableCommandBatching === true;
248
- if (!runtimeBatchingAtDefaults) {
217
+ // Resolve readbackMode. "auto" runs the submit probe and resolves to a
218
+ // concrete mode. Validation runs once here at pipeline init.
219
+ const dl = patch.session.decodeLoop;
220
+ let resolvedReadbackMode = dl.readbackMode;
221
+
222
+ if (resolvedReadbackMode === 'overlapped') {
223
+ const rs = dl.ringStaging ?? 0;
224
+ if (rs < 2) {
225
+ throw new Error(
226
+ `DopplerConfigError: readbackMode "overlapped" requires ringStaging >= 2, got ${rs}.`
227
+ );
228
+ }
229
+ }
230
+
231
+ if (resolvedReadbackMode === 'auto') {
232
+ const thresholdMs = dl.submitLatencyThresholdMs;
233
+ if (thresholdMs == null) {
234
+ throw new Error(
235
+ 'DopplerConfigError: readbackMode "auto" requires submitLatencyThresholdMs to be set.'
236
+ );
237
+ }
238
+ let probeMs = null;
239
+ try {
240
+ probeMs = getKernelCapabilities().submitProbeMs;
241
+ } catch {
242
+ // Device not initialized yet — fall back to sequential.
243
+ }
244
+ if (probeMs != null && probeMs > thresholdMs) {
245
+ const rs = dl.ringStaging ?? 0;
246
+ if (rs < 2) {
247
+ log.info(
248
+ 'Pipeline',
249
+ `readbackMode auto: probe ${probeMs.toFixed(1)}ms > threshold ${thresholdMs}ms ` +
250
+ `but ringStaging ${rs} < 2; staying sequential`
251
+ );
252
+ resolvedReadbackMode = 'sequential';
253
+ } else {
254
+ resolvedReadbackMode = 'overlapped';
255
+ }
256
+ } else {
257
+ resolvedReadbackMode = 'sequential';
258
+ }
249
259
  log.info(
250
260
  'Pipeline',
251
- `Manifest decodeLoop skipped for "${manifest?.modelId ?? 'unknown'}": ` +
252
- 'runtime.inference.batching is already explicitly configured (runtime overrides take precedence).'
253
- );
254
- return runtimeConfig;
255
- }
256
- if (patch.generation && !runtimeGenerationAtDefaults && runtimeDisableCommandBatching !== manifestDisableCommandBatching) {
257
- throw new Error(
258
- 'Manifest decodeLoop.disableCommandBatching conflicts with runtime.inference.generation.disableCommandBatching. ' +
259
- 'Choose one explicit source of truth.'
261
+ `readbackMode resolved to ${resolvedReadbackMode} ` +
262
+ `(probe: ${probeMs != null ? probeMs.toFixed(1) + 'ms' : 'unavailable'}, threshold: ${thresholdMs}ms)`
260
263
  );
261
264
  }
262
265
 
266
+ dl.readbackMode = resolvedReadbackMode;
267
+ patch.batching.readbackMode = resolvedReadbackMode;
268
+
263
269
  const nextRuntimeConfig = mergeRuntimeValues(runtimeConfig, {
264
270
  inference: {
271
+ session: patch.session,
265
272
  batching: patch.batching,
266
- ...(patch.generation ? { generation: patch.generation } : {}),
267
273
  },
268
274
  });
269
275
  log.info(
270
276
  'Pipeline',
271
- `Manifest decodeLoop applied (${manifest?.modelId ?? 'unknown'}): ` +
277
+ `Resolved session applied (${manifest?.modelId ?? 'unknown'}): ` +
272
278
  `batchSize=${patch.batching.batchSize}, stopCheckMode=${patch.batching.stopCheckMode}, ` +
273
279
  `readbackInterval=${patch.batching.readbackInterval}, ` +
274
- `disableCommandBatching=${patch.generation?.disableCommandBatching === true}`
280
+ `disableCommandBatching=${patch.session.decodeLoop?.disableCommandBatching === true}`
275
281
  );
276
282
  return nextRuntimeConfig;
277
283
  }
@@ -365,6 +371,8 @@ function collectKernelPathSteps(kernelPath) {
365
371
  append(kernelPath?.sampling);
366
372
  for (const override of kernelPath?.layerOverrides ?? []) {
367
373
  append(override?.steps);
374
+ append(override?.decode?.steps);
375
+ append(override?.prefill?.steps);
368
376
  }
369
377
  return steps;
370
378
  }
@@ -386,8 +394,6 @@ function findKernelPathUnsupportedFeatureUsages(kernelPath, capabilities) {
386
394
  let supported = true;
387
395
  if (requirement === 'subgroups') {
388
396
  supported = hasSubgroups;
389
- } else if (requirement === 'subgroups-f16') {
390
- supported = hasSubgroups && hasF16;
391
397
  } else if (requirement === 'shader-f16') {
392
398
  supported = hasF16;
393
399
  } else {
@@ -481,13 +487,6 @@ function buildKernelPathDtypeContract(resolvedKernelPath) {
481
487
  };
482
488
  }
483
489
 
484
- function isGlobalKernelPathDtypeDefault(currentValue, key) {
485
- if (currentValue == null) {
486
- return true;
487
- }
488
- return currentValue === GLOBAL_DEFAULT_KERNEL_PATH_DTYPES[key];
489
- }
490
-
491
490
  function describeKernelPathDtypeMismatch(contract, current) {
492
491
  const mismatches = [];
493
492
  if (contract.activationDtype && current.activationDtype !== contract.activationDtype) {
@@ -498,7 +497,7 @@ function describeKernelPathDtypeMismatch(contract, current) {
498
497
  }
499
498
  if (contract.kvDtype && current.kvDtype !== contract.kvDtype) {
500
499
  mismatches.push(
501
- `runtime.inference.kvcache.kvDtype=${current.kvDtype ?? 'unset'} ` +
500
+ `runtime.inference.session.kvcache.kvDtype=${current.kvDtype ?? 'unset'} ` +
502
501
  `(expected ${contract.kvDtype})`
503
502
  );
504
503
  }
@@ -511,6 +510,41 @@ function describeKernelPathDtypeMismatch(contract, current) {
511
510
  return mismatches;
512
511
  }
513
512
 
513
+ function describeExplicitRuntimeDtypeMismatch(contract, explicitRuntime) {
514
+ const mismatches = [];
515
+ if (
516
+ contract.activationDtype
517
+ && explicitRuntime.activationDtype != null
518
+ && explicitRuntime.activationDtype !== contract.activationDtype
519
+ ) {
520
+ mismatches.push(
521
+ `runtime.inference.session.compute.defaults.activationDtype=${explicitRuntime.activationDtype} ` +
522
+ `(expected ${contract.activationDtype})`
523
+ );
524
+ }
525
+ if (
526
+ contract.kvDtype
527
+ && explicitRuntime.kvDtype != null
528
+ && explicitRuntime.kvDtype !== contract.kvDtype
529
+ ) {
530
+ mismatches.push(
531
+ `runtime.inference.session.kvcache.kvDtype=${explicitRuntime.kvDtype} ` +
532
+ `(expected ${contract.kvDtype})`
533
+ );
534
+ }
535
+ if (
536
+ contract.outputDtype
537
+ && explicitRuntime.outputDtype != null
538
+ && explicitRuntime.outputDtype !== contract.outputDtype
539
+ ) {
540
+ mismatches.push(
541
+ `runtime.inference.session.compute.defaults.outputDtype=${explicitRuntime.outputDtype} ` +
542
+ `(expected ${contract.outputDtype})`
543
+ );
544
+ }
545
+ return mismatches;
546
+ }
547
+
514
548
  function assertManifestKernelPathDtypeCompatibility(manifest, resolvedKernelPath, kernelPathSource) {
515
549
  if (!resolvedKernelPath) return;
516
550
  if (kernelPathSource === 'config') return;
@@ -538,9 +572,9 @@ function getKernelCapabilitiesSafe() {
538
572
  }
539
573
 
540
574
  const DEFAULT_KERNEL_PATH_POLICY = Object.freeze({
541
- mode: SCHEMA_DEFAULT_KERNEL_PATH_POLICY.mode,
542
- sourceScope: Object.freeze([...SCHEMA_DEFAULT_KERNEL_PATH_POLICY.sourceScope]),
543
- onIncompatible: SCHEMA_DEFAULT_KERNEL_PATH_POLICY.onIncompatible,
575
+ mode: 'locked',
576
+ sourceScope: Object.freeze(['model', 'manifest']),
577
+ onIncompatible: 'error',
544
578
  });
545
579
 
546
580
  function normalizeKernelPathPolicyMode(value) {
@@ -600,7 +634,28 @@ function resolveKernelPathPolicy(policy) {
600
634
  };
601
635
  }
602
636
 
603
- function applyKernelPathRuntimeDtypeContract(resolvedKernelPath, runtimeConfig, kernelPathSource, modelId) {
637
+ function getExplicitRuntimeDtypeOverrides(runtimeOverrides) {
638
+ return {
639
+ activationDtype: normalizeKernelDtype(
640
+ runtimeOverrides?.inference?.session?.compute?.defaults?.activationDtype
641
+ ?? runtimeOverrides?.inference?.compute?.activationDtype
642
+ ),
643
+ kvDtype: normalizeKernelDtype(
644
+ runtimeOverrides?.inference?.session?.kvcache?.kvDtype
645
+ ),
646
+ outputDtype: normalizeKernelDtype(
647
+ runtimeOverrides?.inference?.session?.compute?.defaults?.outputDtype
648
+ ),
649
+ };
650
+ }
651
+
652
+ function applyKernelPathRuntimeDtypeContract(
653
+ resolvedKernelPath,
654
+ runtimeConfig,
655
+ runtimeOverrides,
656
+ kernelPathSource,
657
+ modelId
658
+ ) {
604
659
  const contract = buildKernelPathDtypeContract(resolvedKernelPath);
605
660
  if (!contract) {
606
661
  return runtimeConfig;
@@ -608,7 +663,7 @@ function applyKernelPathRuntimeDtypeContract(resolvedKernelPath, runtimeConfig,
608
663
 
609
664
  const current = {
610
665
  activationDtype: normalizeKernelDtype(runtimeConfig.inference?.compute?.activationDtype),
611
- kvDtype: normalizeKernelDtype(runtimeConfig.inference?.kvcache?.kvDtype),
666
+ kvDtype: normalizeKernelDtype(runtimeConfig.inference?.session?.kvcache?.kvDtype),
612
667
  outputDtype: normalizeKernelDtype(runtimeConfig.inference?.session?.compute?.defaults?.outputDtype),
613
668
  };
614
669
  const mismatches = describeKernelPathDtypeMismatch(contract, current);
@@ -616,71 +671,42 @@ function applyKernelPathRuntimeDtypeContract(resolvedKernelPath, runtimeConfig,
616
671
  return runtimeConfig;
617
672
  }
618
673
 
674
+ const explicitRuntime = getExplicitRuntimeDtypeOverrides(runtimeOverrides);
675
+ const explicitMismatches = describeExplicitRuntimeDtypeMismatch(contract, explicitRuntime);
676
+
619
677
  if (kernelPathSource === 'config') {
620
678
  throw new Error(
621
679
  `KernelPath "${resolvedKernelPath?.id ?? 'unknown'}" selected from ${kernelPathSource} ` +
622
680
  `requires explicit matching runtime dtypes for "${modelId}". ` +
623
681
  `Mismatches: ${mismatches.join('; ')}. ` +
624
- 'Set runtime.inference.compute.activationDtype, runtime.inference.kvcache.kvDtype, ' +
682
+ 'Set runtime.inference.session.compute.defaults.activationDtype, runtime.inference.session.kvcache.kvDtype, ' +
625
683
  'and runtime.inference.session.compute.defaults.outputDtype to match the kernel path.'
626
684
  );
627
685
  }
628
686
 
629
- const canApplyManifestDefaults = (
630
- (contract.activationDtype == null || isGlobalKernelPathDtypeDefault(current.activationDtype, 'activationDtype'))
631
- && (contract.kvDtype == null || isGlobalKernelPathDtypeDefault(current.kvDtype, 'kvDtype'))
632
- && (contract.outputDtype == null || isGlobalKernelPathDtypeDefault(current.outputDtype, 'outputDtype'))
633
- );
634
- if (!canApplyManifestDefaults) {
687
+ if (explicitMismatches.length > 0) {
635
688
  throw new Error(
636
689
  `Manifest/model kernelPath "${resolvedKernelPath?.id ?? 'unknown'}" for "${modelId}" ` +
637
- `conflicts with runtime dtype overrides. Mismatches: ${mismatches.join('; ')}. ` +
690
+ `conflicts with runtime dtype overrides. Mismatches: ${explicitMismatches.join('; ')}. ` +
638
691
  'Either remove the runtime dtype override or set it to match the kernel path.'
639
692
  );
640
693
  }
641
694
 
642
- const nextInference = {
643
- ...runtimeConfig.inference,
644
- compute: { ...runtimeConfig.inference.compute },
645
- kvcache: { ...runtimeConfig.inference.kvcache },
646
- };
647
- const dtypeChanges = [];
648
-
649
- if (contract.activationDtype && current.activationDtype !== contract.activationDtype) {
650
- nextInference.compute.activationDtype = contract.activationDtype;
651
- dtypeChanges.push(`activation=${current.activationDtype ?? 'unset'}->${contract.activationDtype}`);
652
- }
653
-
654
- if (contract.kvDtype && current.kvDtype !== contract.kvDtype) {
655
- nextInference.kvcache.kvDtype = contract.kvDtype;
656
- dtypeChanges.push(`kv=${current.kvDtype ?? 'unset'}->${contract.kvDtype}`);
657
- }
658
-
659
- if (contract.outputDtype && current.outputDtype !== contract.outputDtype) {
660
- nextInference.session = {
661
- ...(nextInference.session ?? {}),
662
- compute: {
663
- ...(nextInference.session?.compute ?? {}),
664
- defaults: {
665
- ...(nextInference.session?.compute?.defaults ?? {}),
666
- outputDtype: contract.outputDtype,
667
- },
668
- },
669
- };
670
- dtypeChanges.push(`session.outputDtype=${current.outputDtype ?? 'unset'}->${contract.outputDtype}`);
671
- }
672
-
673
- log.info(
674
- 'Pipeline',
675
- `KernelPath ${resolvedKernelPath?.id ?? 'unknown'} applied manifest/model runtime dtype defaults: ${dtypeChanges.join(', ')}`
695
+ throw new Error(
696
+ `Manifest/model kernelPath "${resolvedKernelPath?.id ?? 'unknown'}" for "${modelId}" ` +
697
+ `requires matching runtime dtypes. Mismatches: ${mismatches.join('; ')}. ` +
698
+ 'Set runtime.inference.session.compute.defaults.activationDtype, ' +
699
+ 'runtime.inference.session.kvcache.kvDtype, and ' +
700
+ 'runtime.inference.session.compute.defaults.outputDtype to match the kernel path. ' +
701
+ 'Runtime dtype auto-rewrites are not allowed.'
676
702
  );
677
- return { ...runtimeConfig, inference: nextInference };
678
703
  }
679
704
 
680
705
  export function resolveKernelPathState(options) {
681
706
  const {
682
707
  manifest,
683
708
  runtimeConfig,
709
+ runtimeOverrides = null,
684
710
  modelConfig,
685
711
  kernelCapabilities = null,
686
712
  } = options;
@@ -744,6 +770,7 @@ export function resolveKernelPathState(options) {
744
770
  const nextRuntimeConfig = applyKernelPathRuntimeDtypeContract(
745
771
  resolvedKernelPath,
746
772
  runtimeConfig,
773
+ runtimeOverrides,
747
774
  kernelPathSource,
748
775
  String(manifest?.modelId ?? 'unknown').trim() || 'unknown'
749
776
  );
@@ -25,6 +25,7 @@ import {
25
25
  resolveGptOssKernelPathProfile,
26
26
  resolveMixtralKernelPathProfile,
27
27
  } from './moe-shape-validator.js';
28
+ import { assertImplicitDtypeTransitionAllowed } from './dtype-contract.js';
28
29
 
29
30
  export async function moeFeedForwardGPU(
30
31
  inputBuffer,
@@ -528,6 +529,13 @@ async function runGptOssExpert(
528
529
  let biasTensor = createTensor(weights.gateUpBias, gateUpBiasDtype, [biasElements], 'moe_gate_up_bias');
529
530
  let biasTemp = null;
530
531
  if (biasTensor.dtype !== activationDtype) {
532
+ assertImplicitDtypeTransitionAllowed({
533
+ executionPolicies: config.executionPolicies ?? null,
534
+ fromDtype: biasTensor.dtype,
535
+ toDtype: activationDtype,
536
+ op: 'moe_gate_up_bias',
537
+ detail: `Expert ${expertIdx} gate/up bias would be repacked to match activation dtype.`,
538
+ });
531
539
  biasTemp = activationDtype === 'f16'
532
540
  ? await castF32ToF16(biasTensor)
533
541
  : await castF16ToF32(biasTensor);
@@ -18,6 +18,7 @@ import type { Tensor, TensorDtype } from '../../../gpu/tensor.js';
18
18
  import type { MoERouter } from '../../moe-router.js';
19
19
  import type { ExpertWeights } from './types.js';
20
20
  import type { KernelPathSchema } from '../../../config/schema/index.js';
21
+ import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
21
22
 
22
23
  /**
23
24
  * Clear the dequantization cache (call on model unload).
@@ -53,6 +54,7 @@ export interface MoEConfig {
53
54
  swigluLimit: number | null;
54
55
  activationDtype?: TensorDtype;
55
56
  kernelPath?: KernelPathSchema | null;
57
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
56
58
  }
57
59
 
58
60
  /**
@@ -11,6 +11,7 @@ import type { SiLURowSplitOptions, CommandRecorder } from '../../../gpu/kernel-s
11
11
  import type { Tensor } from '../../../gpu/tensor.js';
12
12
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
13
13
  import type { DecodeBufferManager } from '../../decode-buffers.js';
14
+ import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
14
15
  import type {
15
16
  AttentionConfig,
16
17
  AttentionState,
@@ -49,7 +50,12 @@ export function doResidualAdd(
49
50
  b: Tensor,
50
51
  size: number,
51
52
  recorder?: CommandRecorder,
52
- traceOptions?: { label?: string; layerIdx?: number; outputBuffer?: GPUBuffer | null }
53
+ traceOptions?: {
54
+ label?: string;
55
+ layerIdx?: number;
56
+ outputBuffer?: GPUBuffer | null;
57
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
58
+ }
53
59
  ): Promise<Tensor>;
54
60
 
55
61
  /**
@@ -62,7 +68,14 @@ export function doMatmul(
62
68
  M: number,
63
69
  N: number,
64
70
  K: number,
65
- options?: { transposeB?: boolean | 'auto'; label?: string; layerIdx?: number; outputDtype?: 'f16' | 'f32'; role?: string },
71
+ options?: {
72
+ transposeB?: boolean | 'auto';
73
+ label?: string;
74
+ layerIdx?: number;
75
+ outputDtype?: 'f16' | 'f32';
76
+ role?: string;
77
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
78
+ },
66
79
  recorder?: CommandRecorder
67
80
  ): Promise<Tensor>;
68
81
 
@@ -135,6 +148,7 @@ export function doConv(
135
148
  stride?: number;
136
149
  pad?: number;
137
150
  } | null;
151
+ executionPolicies?: ExecutionV1PoliciesSchema | null;
138
152
  },
139
153
  recorder?: CommandRecorder
140
154
  ): Promise<Tensor>;