@qvac/inference 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (462) hide show
  1. package/NOTICE +89 -41
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +5 -2
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.d.ts +2 -1
  9. package/dist/api/completion-stream.js +4 -2
  10. package/dist/api/decide.d.ts +26 -0
  11. package/dist/api/decide.js +26 -0
  12. package/dist/api/diffusion.d.ts +18 -1
  13. package/dist/api/diffusion.js +18 -1
  14. package/dist/api/download-asset.js +1 -1
  15. package/dist/api/finetune.js +1 -1
  16. package/dist/api/get-model-info.d.ts +1 -1
  17. package/dist/api/index.d.ts +39 -37
  18. package/dist/api/index.js +38 -36
  19. package/dist/api/load-model.js +2 -2
  20. package/dist/api/logging-stream-registry.js +11 -7
  21. package/dist/api/logging-stream.d.ts +11 -1
  22. package/dist/api/logging-stream.js +11 -2
  23. package/dist/api/registry.d.ts +2 -1
  24. package/dist/api/registry.js +5 -4
  25. package/dist/api/rerank.d.ts +17 -0
  26. package/dist/api/rerank.js +36 -0
  27. package/dist/api/subscribe-logs.js +5 -3
  28. package/dist/api/text-to-speech.d.ts +1 -0
  29. package/dist/api/text-to-speech.js +15 -19
  30. package/dist/api/transcribe.js +5 -2
  31. package/dist/api/unload-model.js +1 -1
  32. package/dist/api/video-request.d.ts +5 -5
  33. package/dist/api/video.d.ts +5 -4
  34. package/dist/api/video.js +6 -5
  35. package/dist/api/vla.js +1 -1
  36. package/dist/api/world-result.js +1 -1
  37. package/dist/api/world.d.ts +1 -1
  38. package/dist/api/world.js +1 -1
  39. package/dist/config/resolve-config.d.ts +1 -1
  40. package/dist/config/resolve-config.js +1 -1
  41. package/dist/constants/index.d.ts +2 -2
  42. package/dist/constants/index.js +2 -2
  43. package/dist/dispatch.d.ts +3 -2
  44. package/dist/dispatch.js +16 -4
  45. package/dist/errors/index.d.ts +9 -0
  46. package/dist/errors/index.js +18 -1
  47. package/dist/handlers/assess-model-fit.d.ts +5 -3
  48. package/dist/handlers/assess-model-fit.js +84 -32
  49. package/dist/handlers/cancelHandler.js +1 -1
  50. package/dist/handlers/completion-orchestrate.js +2 -2
  51. package/dist/handlers/download-asset.js +2 -2
  52. package/dist/handlers/finetune.js +1 -1
  53. package/dist/handlers/get-loaded-model-info.js +3 -5
  54. package/dist/handlers/get-model-info.js +1 -1
  55. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
  56. package/dist/handlers/load-model/coreml-sidecars.js +71 -8
  57. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  58. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  59. package/dist/handlers/load-model/download-stats.js +3 -3
  60. package/dist/handlers/load-model/handler.js +10 -7
  61. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  62. package/dist/handlers/load-model/http.d.ts +1 -1
  63. package/dist/handlers/load-model/http.js +3 -3
  64. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  65. package/dist/handlers/load-model/hyperdrive.js +2 -2
  66. package/dist/handlers/load-model/index.d.ts +2 -2
  67. package/dist/handlers/load-model/index.js +2 -2
  68. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  69. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  70. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  71. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  72. package/dist/handlers/load-model/registry.d.ts +2 -2
  73. package/dist/handlers/load-model/registry.js +5 -5
  74. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  75. package/dist/handlers/load-model/resolve-session.js +11 -3
  76. package/dist/handlers/load-model/resolve.d.ts +1 -1
  77. package/dist/handlers/load-model/resolve.js +5 -5
  78. package/dist/handlers/load-model/safe-fetch.js +5 -6
  79. package/dist/handlers/logging-stream.d.ts +2 -1
  80. package/dist/handlers/logging-stream.js +23 -7
  81. package/dist/handlers/registry.d.ts +3 -1
  82. package/dist/handlers/registry.js +7 -5
  83. package/dist/handlers/types.d.ts +7 -1
  84. package/dist/handlers/vector-index.js +2 -2
  85. package/dist/index.d.ts +1 -1
  86. package/dist/index.js +1 -1
  87. package/dist/logging/addon.d.ts +1 -1
  88. package/dist/logging/addon.js +1 -1
  89. package/dist/logging/index.d.ts +5 -5
  90. package/dist/logging/index.js +4 -4
  91. package/dist/logging/logger.d.ts +1 -1
  92. package/dist/logging/logger.js +2 -2
  93. package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
  94. package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
  95. package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
  96. package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
  97. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
  98. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
  99. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
  100. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
  101. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
  102. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
  103. package/dist/model-fit/floor.d.ts +31 -0
  104. package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
  105. package/dist/model-fit/kv-cache.d.ts +23 -0
  106. package/dist/model-fit/kv-cache.js +85 -0
  107. package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
  108. package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
  109. package/dist/model-fit/platform.d.ts +14 -0
  110. package/dist/model-fit/platform.js +26 -0
  111. package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
  112. package/dist/model-fit/request/create.js +61 -0
  113. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
  114. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
  115. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
  116. package/dist/model-fit/request/engines/audiogen.js +63 -0
  117. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
  118. package/dist/model-fit/request/engines/diffusion.js +93 -0
  119. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
  120. package/dist/model-fit/request/engines/llama.js +44 -0
  121. package/dist/model-fit/request/engines/nmt.d.ts +8 -0
  122. package/dist/model-fit/request/engines/nmt.js +11 -0
  123. package/dist/model-fit/request/engines/ocr.d.ts +9 -0
  124. package/dist/model-fit/request/engines/ocr.js +13 -0
  125. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
  126. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
  127. package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
  128. package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
  129. package/dist/model-fit/run/run.d.ts +43 -0
  130. package/dist/model-fit/run/run.js +204 -0
  131. package/dist/models/registry/index.d.ts +1 -1
  132. package/dist/models/registry/index.js +2 -2
  133. package/dist/models/registry/models.d.ts +1072 -815
  134. package/dist/models/registry/models.js +3704 -3405
  135. package/dist/models/registry/resource-profiles.d.ts +1 -2
  136. package/dist/models/registry/resource-profiles.js +406 -3
  137. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  138. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  139. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  140. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  141. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
  142. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  143. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  144. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
  145. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
  146. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
  147. package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
  148. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
  149. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
  150. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  151. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  152. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  153. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
  154. package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
  155. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
  156. package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
  157. package/dist/plugins/builtin/index.d.ts +15 -12
  158. package/dist/plugins/builtin/index.js +15 -12
  159. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
  160. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  161. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  162. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
  163. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
  164. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
  165. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
  166. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  167. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  168. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
  169. package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
  170. package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
  171. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
  172. package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
  173. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  174. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  175. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
  176. package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
  177. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
  178. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
  179. package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
  180. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  181. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  182. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
  183. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
  184. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
  185. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
  186. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  187. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  188. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  189. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  190. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
  191. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
  192. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
  193. package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
  194. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
  195. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
  196. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
  197. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
  198. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
  199. package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
  200. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  201. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  202. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  203. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  204. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  206. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
  207. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
  208. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
  209. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
  210. package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
  211. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
  212. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
  213. package/dist/plugins/index.d.ts +1 -1
  214. package/dist/plugins/index.js +2 -2
  215. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
  216. package/dist/plugins/ops/kv-cache-utils.js +25 -33
  217. package/dist/plugins/ops/load-model.d.ts +8 -2
  218. package/dist/plugins/ops/load-model.js +33 -19
  219. package/dist/plugins/ops/transcribe.js +18 -2
  220. package/dist/plugins/ops/translate.js +3 -3
  221. package/dist/plugins/ops/unload-model.js +1 -1
  222. package/dist/plugins/turbovec-backend.js +5 -5
  223. package/dist/profiling/aggregator.d.ts +1 -1
  224. package/dist/profiling/aggregator.js +1 -1
  225. package/dist/profiling/controller.d.ts +1 -1
  226. package/dist/profiling/controller.js +2 -2
  227. package/dist/profiling/events.d.ts +1 -1
  228. package/dist/profiling/events.js +2 -2
  229. package/dist/profiling/exporters.d.ts +1 -1
  230. package/dist/profiling/exporters.js +3 -3
  231. package/dist/profiling/index.d.ts +13 -13
  232. package/dist/profiling/index.js +12 -12
  233. package/dist/profiling/operation-metrics.d.ts +1 -1
  234. package/dist/profiling/operation-metrics.js +2 -2
  235. package/dist/profiling/operation-wrappers.js +3 -3
  236. package/dist/profiling/profiler.js +1 -1
  237. package/dist/rag/handler.js +1 -1
  238. package/dist/rag/handlers/close-workspace.js +1 -1
  239. package/dist/rag/handlers/delete-embeddings.js +1 -1
  240. package/dist/rag/handlers/delete-workspace.js +1 -1
  241. package/dist/rag/handlers/ingest.js +1 -1
  242. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  243. package/dist/rag/handlers/list-workspaces.js +1 -1
  244. package/dist/rag/handlers/reindex.js +1 -1
  245. package/dist/rag/handlers/save-embeddings.js +1 -1
  246. package/dist/rag/handlers/search.js +1 -1
  247. package/dist/rag/index.d.ts +11 -11
  248. package/dist/rag/index.js +11 -11
  249. package/dist/rag/rag-workspace-manager.js +1 -1
  250. package/dist/registry.js +3 -1
  251. package/dist/resources/collector.d.ts +1 -1
  252. package/dist/resources/collector.js +1 -1
  253. package/dist/resources/instance.d.ts +1 -1
  254. package/dist/resources/instance.js +1 -1
  255. package/dist/resources/native.d.ts +1 -1
  256. package/dist/resources/normalize.d.ts +1 -1
  257. package/dist/rpc/inventory.js +5 -7
  258. package/dist/runtime/env.d.ts +0 -2
  259. package/dist/runtime/env.js +2 -9
  260. package/dist/runtime/index.d.ts +5 -5
  261. package/dist/runtime/index.js +2 -2
  262. package/dist/runtime/lifecycle.js +7 -7
  263. package/dist/runtime/model-config-utils.js +9 -1
  264. package/dist/runtime/model-registry.d.ts +3 -1
  265. package/dist/runtime/model-registry.js +2 -1
  266. package/dist/runtime/registry-client.js +1 -1
  267. package/dist/runtime/request-context.d.ts +3 -3
  268. package/dist/runtime/request-context.js +22 -1
  269. package/dist/runtime/request-registry.d.ts +1 -1
  270. package/dist/runtime/request-registry.js +1 -1
  271. package/dist/runtime/state.d.ts +1 -1
  272. package/dist/runtime/state.js +7 -2
  273. package/dist/runtime/vector-index-backend.d.ts +2 -2
  274. package/dist/runtime/vector-index-registry.d.ts +5 -5
  275. package/dist/runtime/vector-index-registry.js +36 -13
  276. package/dist/schemas/assess-model-fit.d.ts +106 -43
  277. package/dist/schemas/assess-model-fit.js +69 -32
  278. package/dist/schemas/audio-gen.d.ts +199 -19
  279. package/dist/schemas/audio-gen.js +183 -51
  280. package/dist/schemas/batch-completion-stream.d.ts +210 -36
  281. package/dist/schemas/batch-completion-stream.js +3 -3
  282. package/dist/schemas/bci-config.d.ts +3 -0
  283. package/dist/schemas/bci-config.js +2 -2
  284. package/dist/schemas/bci.d.ts +24 -3
  285. package/dist/schemas/bci.js +2 -2
  286. package/dist/schemas/cancel.d.ts +10 -0
  287. package/dist/schemas/cancel.js +2 -0
  288. package/dist/schemas/common.d.ts +1465 -162
  289. package/dist/schemas/common.js +37 -31
  290. package/dist/schemas/completion-event.d.ts +8 -2
  291. package/dist/schemas/completion-event.js +4 -1
  292. package/dist/schemas/completion-stream.d.ts +243 -39
  293. package/dist/schemas/completion-stream.js +100 -29
  294. package/dist/schemas/config.d.ts +1326 -18
  295. package/dist/schemas/config.js +30 -8
  296. package/dist/schemas/decision.d.ts +610 -0
  297. package/dist/schemas/decision.js +219 -0
  298. package/dist/schemas/download-asset.d.ts +10 -4
  299. package/dist/schemas/download-asset.js +1 -1
  300. package/dist/schemas/engine-addon-map.d.ts +9 -1
  301. package/dist/schemas/engine-addon-map.js +26 -5
  302. package/dist/schemas/errors.d.ts +2 -0
  303. package/dist/schemas/errors.js +11 -1
  304. package/dist/schemas/index.d.ts +51 -48
  305. package/dist/schemas/index.js +50 -47
  306. package/dist/schemas/llamacpp-config.d.ts +222 -6
  307. package/dist/schemas/llamacpp-config.js +129 -7
  308. package/dist/schemas/load-model.d.ts +7960 -1492
  309. package/dist/schemas/load-model.js +122 -10
  310. package/dist/schemas/mcp-adapter.d.ts +1 -1
  311. package/dist/schemas/model-config-schemas.d.ts +428 -9
  312. package/dist/schemas/model-config-schemas.js +16 -11
  313. package/dist/schemas/model-fit-policy.d.ts +17 -0
  314. package/dist/schemas/model-fit-policy.js +17 -0
  315. package/dist/schemas/model-info.d.ts +20 -2
  316. package/dist/schemas/model-info.js +4 -3
  317. package/dist/schemas/model-resource-profile.d.ts +4 -4
  318. package/dist/schemas/model-resource-profile.js +2 -5
  319. package/dist/schemas/model-src-utils.d.ts +14 -2
  320. package/dist/schemas/model-src-utils.js +2 -2
  321. package/dist/schemas/model-types.d.ts +14 -1
  322. package/dist/schemas/model-types.js +4 -0
  323. package/dist/schemas/ocr.d.ts +76 -0
  324. package/dist/schemas/ocr.js +43 -1
  325. package/dist/schemas/plugin.d.ts +16 -4
  326. package/dist/schemas/plugin.js +12 -0
  327. package/dist/schemas/profiling.d.ts +4 -4
  328. package/dist/schemas/profiling.js +1 -1
  329. package/dist/schemas/rag.js +1 -1
  330. package/dist/schemas/registry.d.ts +34 -0
  331. package/dist/schemas/registry.js +5 -1
  332. package/dist/schemas/reload-config.d.ts +14 -2
  333. package/dist/schemas/reload-config.js +2 -2
  334. package/dist/schemas/rerank.d.ts +131 -0
  335. package/dist/schemas/rerank.js +102 -0
  336. package/dist/schemas/rpc-server.js +3 -3
  337. package/dist/schemas/sdcpp-config.d.ts +82 -0
  338. package/dist/schemas/sdcpp-config.js +41 -22
  339. package/dist/schemas/system-resources.d.ts +4 -4
  340. package/dist/schemas/text-to-speech.d.ts +463 -3
  341. package/dist/schemas/text-to-speech.js +129 -11
  342. package/dist/schemas/tools.d.ts +13 -4
  343. package/dist/schemas/tools.js +18 -6
  344. package/dist/schemas/transcription-config.d.ts +20 -0
  345. package/dist/schemas/transcription-config.js +54 -4
  346. package/dist/schemas/transcription.d.ts +47 -5
  347. package/dist/schemas/transcription.js +52 -5
  348. package/dist/schemas/translate.d.ts +2 -2
  349. package/dist/schemas/translate.js +1 -1
  350. package/dist/schemas/translation-config.d.ts +40 -10
  351. package/dist/schemas/translation-config.js +1 -1
  352. package/dist/schemas/vla.d.ts +9 -2
  353. package/dist/schemas/vla.js +9 -5
  354. package/dist/surface.d.ts +3 -2
  355. package/dist/surface.js +3 -2
  356. package/dist/utils/addon-responses.d.ts +3 -0
  357. package/dist/utils/aggregate-events.d.ts +1 -1
  358. package/dist/utils/aggregate-events.js +2 -2
  359. package/dist/utils/archive.js +2 -2
  360. package/dist/utils/asr-events.d.ts +1 -0
  361. package/dist/utils/asr-events.js +2 -1
  362. package/dist/utils/audio/decoder.js +3 -1
  363. package/dist/utils/audio-input.js +1 -1
  364. package/dist/utils/cache/auto-history.d.ts +2 -1
  365. package/dist/utils/cache/auto-history.js +2 -2
  366. package/dist/utils/cache/paths.js +6 -6
  367. package/dist/utils/cache/types.d.ts +7 -0
  368. package/dist/utils/checksum.d.ts +2 -0
  369. package/dist/utils/checksum.js +41 -2
  370. package/dist/utils/completion-normalizer.js +1 -1
  371. package/dist/utils/gguf-architecture.d.ts +7 -0
  372. package/dist/utils/gguf-architecture.js +142 -0
  373. package/dist/utils/gguf-tensor-extractor.js +1 -1
  374. package/dist/utils/index.d.ts +13 -13
  375. package/dist/utils/index.js +13 -13
  376. package/dist/utils/mcp-adapter.d.ts +1 -1
  377. package/dist/utils/mcp-adapter.js +13 -9
  378. package/dist/utils/path-security.js +1 -1
  379. package/dist/utils/shard-utils.d.ts +5 -0
  380. package/dist/utils/shard-utils.js +19 -3
  381. package/dist/utils/tool-helpers.d.ts +1 -1
  382. package/dist/utils/tool-helpers.js +20 -52
  383. package/dist/utils/tool-integration.d.ts +1 -0
  384. package/dist/utils/tool-integration.js +3 -6
  385. package/dist/utils/tools/defer.d.ts +16 -26
  386. package/dist/utils/tools/defer.js +50 -41
  387. package/dist/utils/tools/dialect.d.ts +15 -4
  388. package/dist/utils/tools/dialect.js +34 -5
  389. package/dist/utils/tools/index.d.ts +3 -3
  390. package/dist/utils/tools/index.js +3 -3
  391. package/dist/utils/tools/parser.js +8 -8
  392. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  393. package/dist/utils/tools/parsers/dsml.js +8 -5
  394. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  395. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  396. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  397. package/dist/utils/tools/parsers/harmony.js +1 -1
  398. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  399. package/dist/utils/tools/parsers/hermes.js +1 -1
  400. package/dist/utils/tools/parsers/json.d.ts +1 -1
  401. package/dist/utils/tools/parsers/json.js +1 -1
  402. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  403. package/dist/utils/tools/parsers/pythonic.js +1 -1
  404. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  405. package/dist/utils/tools/parsers/qwen35.js +7 -4
  406. package/dist/utils/tools/shared.d.ts +3 -0
  407. package/dist/utils/tools/shared.js +62 -0
  408. package/dist/utils/transcribe-metadata.d.ts +8 -1
  409. package/dist/utils/transcribe-metadata.js +13 -3
  410. package/dist/utils/tts-stats.d.ts +2 -1
  411. package/dist/utils/until-aborted.d.ts +7 -0
  412. package/dist/utils/until-aborted.js +34 -0
  413. package/package.json +68 -51
  414. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  415. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  416. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  417. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  418. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  419. package/dist/resources/model-fit/calibration/fit.js +0 -152
  420. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  421. package/dist/resources/model-fit/calibration/index.js +0 -75
  422. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  423. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  424. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  425. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  426. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  427. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  428. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  429. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  430. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  431. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  432. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  433. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  434. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  435. package/dist/resources/model-fit/estimators/llm.js +0 -222
  436. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  437. package/dist/resources/model-fit/estimators/whisper.js +0 -76
  438. package/dist/resources/model-fit/floor.d.ts +0 -31
  439. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  440. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
  441. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
  442. package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
  443. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
  444. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
  445. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  446. package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
  447. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  448. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  449. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  450. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  451. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  452. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  453. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
  454. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
  455. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  456. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  457. package/dist/resources/model-fit/platform.d.ts +0 -9
  458. package/dist/resources/model-fit/platform.js +0 -28
  459. package/dist/resources/model-fit/types.d.ts +0 -120
  460. package/dist/resources/model-fit/types.js +0 -1
  461. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
  462. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
@@ -1,15 +1,6 @@
1
- import { getModelResourceProfile } from '../../models/registry/resource-profiles.js';
2
- import { getGpuCalibration, getSharedGpuCalibration } from './calibration/index.js';
3
- import { estimateLlm } from './estimators/llm.js';
4
- import { estimateWhisper } from './estimators/whisper.js';
5
- import { computeFloor, FLOOR_VERSION } from './floor.js';
1
+ import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
2
+ import { computeFloor, FLOOR_VERSION } from '../model-fit/floor.js';
6
3
  const GIB = 1024 * 1024 * 1024;
7
- /** Engines phase 1 can estimate, and which estimator owns each. */
8
- const ESTIMATORS = {
9
- 'llamacpp-completion': estimateLlm,
10
- 'llamacpp-embedding': estimateLlm,
11
- 'whispercpp-transcription': estimateWhisper
12
- };
13
4
  const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
14
5
  // Every allocation, GPU included, is system RAM: the system budget always bounds the load.
15
6
  const UNIFIED_MEMORY_PLATFORMS = [
@@ -18,44 +9,53 @@ const UNIFIED_MEMORY_PLATFORMS = [
18
9
  'ios-arm64'
19
10
  ];
20
11
  /**
21
- * Platforms whose fixture describes CPU-resident execution (`device: 'cpu'`),
22
- * so a reported GPU sends the host through `resolveGpuPlacement` instead.
23
- * Apple silicon and mobile calibrate on their GPU already.
12
+ * What one fitter answered: its measurement where the figures divide, its bare
13
+ * verdict where they do not, and `undefined` where it reached no verdict.
24
14
  */
25
- const CPU_CALIBRATED_PLATFORMS = [
26
- 'darwin-x64',
27
- 'linux-arm64',
28
- 'linux-x64',
29
- 'win32-x64'
30
- ];
15
+ function nativeEvaluation(native) {
16
+ const fit = native?.fit;
17
+ if (!fit || fit.verdict === 'unknown')
18
+ return undefined;
19
+ const reasons = fit.message === undefined ? [fit.reason] : [fit.reason, fit.message];
20
+ const bare = {
21
+ kind: 'bare',
22
+ verdict: fit.verdict,
23
+ estimatorVersion: fit.estimatorVersion,
24
+ reasons
25
+ };
26
+ const projection = fit.projection;
27
+ if (!projection)
28
+ return bare;
29
+ const { weightsBytes, contextBytes, computeBytes, hostBytes } = projection;
30
+ if (weightsBytes === undefined || contextBytes === undefined || computeBytes === undefined) {
31
+ return bare;
32
+ }
33
+ return {
34
+ kind: 'native',
35
+ persistent: weightsBytes + contextBytes,
36
+ working: computeBytes,
37
+ hostBytes: hostBytes ?? 0,
38
+ estimatorVersion: fit.estimatorVersion,
39
+ verdict: fit.verdict,
40
+ reasons
41
+ };
42
+ }
31
43
  /**
32
44
  * Turns catalog profiles plus a fresh system-memory sample into an advisory
33
45
  * verdict.
34
46
  *
35
47
  * Pure: every input is passed in, so the same call is testable without a worker,
36
- * a device, or a network. Sampling, platform detection and calibration lookup
37
- * happen in the handler.
48
+ * a device, or a network. Sampling, platform detection and the engine fitter
49
+ * calls happen in the handler.
38
50
  */
39
51
  export function assessModelFitFromResources(options) {
40
- const { models, execution, resources, platform, calibration } = options;
52
+ const { models, execution, resources, platform } = options;
41
53
  const resolveProfile = options.resolveProfile ?? getModelResourceProfile;
42
- const resolveGpuCalibration = options.resolveGpuCalibration ?? getGpuCalibration;
43
- const resolveSharedGpuCalibration = options.resolveSharedGpuCalibration ?? getSharedGpuCalibration;
44
- // Where the model would execute decides which evidence can bound it, and
45
- // which fixture describes the load. See METHODOLOGY.md, "Which fixture a
46
- // host gets".
47
- const mustPlaceGpu = platform !== undefined && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources);
48
- const placement = mustPlaceGpu && platform ? resolveGpuPlacement(resources, platform) : undefined;
49
- const gpuCalibration = placement && platform
50
- ? placement.kind === 'device'
51
- ? resolveGpuCalibration(platform, placement.backend)
52
- : resolveSharedGpuCalibration(platform, placement.backend)
54
+ const placement = platform !== undefined && hasGpu(resources)
55
+ ? resolveGpuPlacement(resources, platform)
53
56
  : undefined;
54
- // A mode only engages with coefficients measured for it.
55
- const onDevice = placement?.kind === 'device' && gpuCalibration ? placement : undefined;
56
- const onIntegrated = placement?.kind === 'shared' && gpuCalibration ? placement : undefined;
57
- const gpuMode = onDevice !== undefined || onIntegrated !== undefined;
58
- const effectiveCalibration = gpuMode ? gpuCalibration : calibration;
57
+ const onDevice = placement?.kind === 'device' ? placement : undefined;
58
+ const onIntegrated = placement?.kind === 'shared' ? placement : undefined;
59
59
  const basis = onDevice
60
60
  ? onDevice.targets[0].scope === 'budget'
61
61
  ? 'device-budget'
@@ -87,106 +87,86 @@ export function assessModelFitFromResources(options) {
87
87
  // A discrete-GPU load is paid for in system RAM too. In shared mode the
88
88
  // system budget already *is* the budget.
89
89
  const alsoBoundBy = onDevice ? resolveBudget(resources, platform, 'system-memory', []) : undefined;
90
+ // The computed floor says something only where the model executes out of the
91
+ // memory the budget measures. A discrete card holds the weights in its own
92
+ // memory, so the system budget bounds nothing there.
93
+ const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
94
+ const evaluated = models.map((candidate, index) => evaluate(candidate, resolveProfile, floorApplies, options.nativeFits?.[index], models.length === 1));
95
+ const results = evaluated.map(({ result }) => result);
90
96
  if (!platform) {
91
97
  reasons.push('the runtime platform is not one this assessment covers');
92
98
  }
93
- else if (mustPlaceGpu && !gpuMode) {
94
- // Name the missing fixture: the platform's own coefficients exist here,
95
- // they are just the wrong ones.
96
- reasons.push(!placement
97
- ? 'a GPU is reported but its readings cannot say where the model would execute, so no estimate can be defended'
98
- : placement.kind === 'device'
99
- ? `no validated calibration for ${platform} on ${placement.backend}, so no estimate can be defended`
100
- : `no validated calibration for ${platform} on an integrated ${placement.backend} GPU, so no estimate can be defended`);
101
- }
102
- else if (!effectiveCalibration) {
103
- reasons.push(`no validated calibration for ${platform}, so no estimate can be defended`);
104
- }
105
- else if (effectiveCalibration.measuredAt) {
106
- assumptions.push(calibrationAssumption(platform, effectiveCalibration));
107
- }
108
- // Without coefficients, the computed floor still says something — but only
109
- // where the model executes out of the memory the budget measures. A discrete
110
- // card holds the weights in its own memory, so the system budget bounds
111
- // nothing there and the floor has no budget to be compared against.
112
- const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
113
- const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
114
99
  for (const { result } of evaluated) {
100
+ if (result.kind === 'bare')
101
+ continue;
115
102
  for (const assumption of result.assumptions ?? []) {
116
103
  if (!assumptions.includes(assumption))
117
104
  assumptions.push(assumption);
118
105
  }
119
106
  }
120
107
  const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
121
- const results = evaluated.map(({ result }) => result);
122
- const estimates = results.filter((result) => {
123
- return result.kind === 'estimate';
124
- });
108
+ const natives = results.filter((result) => result.kind === 'native');
125
109
  const anyUnknown = results.some((result) => result.kind === 'unknown');
126
110
  const anyFloor = results.some((result) => result.kind === 'floor');
127
- // A two-sided bound needs every candidate estimated; a floor needs every
128
- // candidate to have at least a floor. One model with neither leaves the set
129
- // with no evidence at all.
130
- const combined = anyUnknown || anyFloor ? undefined : aggregate(estimates, execution);
131
- const combinedFloor = anyUnknown ? undefined : aggregateFloor(results, execution);
132
- // Absent when a candidate rests on nothing: the set then has no single kind
133
- // of evidence to name, and an `unknown` with `evidence` present stays a
134
- // near-miss or an uncalibrated floor rather than a missing model.
135
- const evidence = anyUnknown
136
- ? undefined
137
- : anyFloor
138
- ? 'computed-only'
139
- : estimates.length > 0
140
- ? 'calibration'
141
- : undefined;
111
+ const bares = results.filter((result) => result.kind === 'bare');
112
+ const allNative = natives.length === results.length && natives.length > 0;
113
+ // Measured bytes are exact, so a set of them can confirm a fit. Mixed with a
114
+ // floor the total is a lower bound again, and `combinedFloor` carries it.
115
+ const combinedNative = allNative ? aggregateNative(natives, execution) : undefined;
116
+ const combinedFloor = anyUnknown || bares.length > 0 ? undefined : aggregateFloor(results, execution);
117
+ // A bare verdict is the engine's own, so it names the evidence; otherwise the
118
+ // weakest candidate does. Absent when a candidate rests on nothing, so an
119
+ // `unknown` carrying `evidence` is a near-miss rather than a missing model.
120
+ const evidence = bares.length > 0
121
+ ? 'native-fit'
122
+ : anyUnknown
123
+ ? undefined
124
+ : anyFloor
125
+ ? 'computed-only'
126
+ : allNative
127
+ ? 'native-fit'
128
+ : undefined;
142
129
  if (anyUnknown) {
143
- reasons.push('at least one model could not be estimated, so the combined verdict is unknown');
144
- }
145
- const verdict = !budget
146
- ? 'unknown'
147
- : combined
148
- ? verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy)
149
- : combinedFloor !== undefined
150
- ? floorVerdict(combinedFloor, deviceBudgets ?? [budget])
151
- : 'unknown';
152
- if (onDevice && combined && alsoBoundBy) {
130
+ reasons.push('at least one model could not be assessed, so the combined verdict is unknown');
131
+ }
132
+ // An engine refuses for reasons its own figures do not carry — no device
133
+ // could hold the placement, a load shape it will not run — so its refusal
134
+ // stands whatever the arithmetic says.
135
+ const refused = bares.some((bare) => bare.verdict === 'does-not-fit') ||
136
+ natives.some((native) => native.verdict === 'does-not-fit');
137
+ // The engine measured this model against this device, so its verdict stands
138
+ // where the arithmetic cannot run. A set still needs a budget, since each
139
+ // model was measured alone.
140
+ const engineAnswered = bares.length > 0 || (!budget && models.length === 1 && natives.length === 1);
141
+ const verdict = refused
142
+ ? 'likely-too-large'
143
+ : engineAnswered
144
+ ? 'likely-fits'
145
+ : !budget
146
+ ? 'unknown'
147
+ : combinedNative !== undefined
148
+ ? verdictAgainst(combinedNative, deviceBudgets ?? [budget], alsoBoundBy)
149
+ : combinedFloor !== undefined
150
+ ? floorVerdict(combinedFloor, deviceBudgets ?? [budget], alsoBoundBy)
151
+ : 'unknown';
152
+ if (onDevice && combinedNative !== undefined && alsoBoundBy) {
153
153
  reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
154
154
  }
155
- if (deviceBudgets && deviceBudgets.length > 1 && combined) {
155
+ if (deviceBudgets && deviceBudgets.length > 1 && combinedNative !== undefined) {
156
156
  reasons.push('more than one usable GPU is reported, so a fit has to hold on the smallest of them and a refusal on the largest');
157
157
  }
158
- if (budget && combined) {
158
+ if (budget && (combinedNative !== undefined || combinedFloor !== undefined)) {
159
159
  reasons.push(execution === 'concurrent'
160
160
  ? 'all models counted as resident with every working peak added'
161
161
  : 'all models counted as resident with only the largest working peak added');
162
162
  }
163
- else if (budget && combinedFloor !== undefined) {
164
- reasons.push(execution === 'concurrent'
165
- ? 'all models counted as resident with their floors summed and every calibrated working peak added'
166
- : 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
167
- }
168
163
  if (anyFloor) {
169
164
  reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
170
165
  }
171
- // The engine's own fitter, where it reached a verdict, replaces the modelled
172
- // one: it read this model's tensor list and measured this machine, which is
173
- // what the coefficients above approximate. The estimator's reasons and
174
- // assumptions describe the path not taken, so none of them are reported.
175
- const native = nativeModelResult(options.nativeFit, modelResults);
176
- if (native) {
177
- return {
178
- verdict: native.verdict,
179
- basis,
180
- execution,
181
- evidence: 'native-fit',
182
- ...(budget && { budget }),
183
- models: [native],
184
- reasons: ['the engine fitter read the registry description of this model'],
185
- assumptions: []
186
- };
166
+ if (allNative) {
167
+ reasons.push('every model was measured by the engine that would run it');
187
168
  }
188
- const declined = nativeFitDecline(options.nativeFit, options.nativeFitUnavailable);
189
- if (declined !== undefined) {
169
+ for (const declined of nativeFitDeclines(options.nativeFits, models)) {
190
170
  reasons.push(declined);
191
171
  }
192
172
  return {
@@ -195,54 +175,31 @@ export function assessModelFitFromResources(options) {
195
175
  execution,
196
176
  ...(evidence && { evidence }),
197
177
  ...(budget && { budget }),
198
- ...(combined && {
199
- estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
200
- }),
201
- ...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
178
+ ...(anyFloor && combinedFloor !== undefined && { floorBytes: totalBytes(combinedFloor) }),
202
179
  models: modelResults,
203
180
  reasons,
204
181
  assumptions
205
182
  };
206
183
  }
207
184
  /**
208
- * Why no engine verdict backs this assessment, for the cases where one was
209
- * sought: the fitter never ran, or it ran and declined. `undefined` where none
210
- * was sought, which is every multi-candidate assessment.
211
- */
212
- function nativeFitDecline(nativeFit, unavailable) {
213
- if (unavailable !== undefined)
214
- return `no engine fit: ${unavailable}`;
215
- if (nativeFit?.verdict !== 'unknown')
216
- return undefined;
217
- const detail = nativeFit.message === undefined ? nativeFit.reason : nativeFit.message;
218
- return `the engine fitter reached no verdict: ${detail}`;
219
- }
220
- /**
221
- * The per-model result a native fit produces, or `undefined` when the modelled
222
- * one stands.
223
- *
224
- * The probe answers about one model against the whole machine. Two such answers
225
- * carry no byte demand that could be summed under one budget, so a multi-model
226
- * assessment keeps the estimator that can aggregate. `unknown` — the probe
227
- * disabled, the load shape unsupported, the child unusable — is not a verdict
228
- * and falls through the same way.
185
+ * Why a candidate carries no engine verdict: the fitter never ran for it, or it
186
+ * ran and declined. Each is named with its model, since a set can lose one
187
+ * model to the fitter and keep the rest.
229
188
  */
230
- function nativeModelResult(nativeFit, modelResults) {
231
- if (!nativeFit || modelResults.length !== 1)
232
- return undefined;
233
- if (nativeFit.verdict === 'unknown')
234
- return undefined;
235
- const modelled = modelResults[0];
236
- if (!modelled)
237
- return undefined;
238
- return {
239
- name: modelled.name,
240
- verdict: nativeFit.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
241
- evidence: 'native-fit',
242
- ...(modelled.device !== undefined && { device: modelled.device }),
243
- estimatorVersion: nativeFit.estimatorVersion,
244
- reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
245
- };
189
+ function nativeFitDeclines(nativeFits, models) {
190
+ const declines = [];
191
+ nativeFits?.forEach((native, index) => {
192
+ const name = models[index]?.model.name ?? `model ${index + 1}`;
193
+ if (native.unavailable !== undefined) {
194
+ declines.push(`no engine fit for ${name}: ${native.unavailable}`);
195
+ return;
196
+ }
197
+ if (native.fit?.verdict !== 'unknown')
198
+ return;
199
+ const detail = native.fit.message === undefined ? native.fit.reason : native.fit.message;
200
+ declines.push(`the engine fitter reached no verdict for ${name}: ${detail}`);
201
+ });
202
+ return declines;
246
203
  }
247
204
  /**
248
205
  * Whether every allocation a load makes comes out of the memory the system (or
@@ -266,13 +223,16 @@ export function boundBySystemMemory(resources, platform) {
266
223
  return gpus.every(allocatesFromSystemMemory);
267
224
  }
268
225
  /**
269
- * One candidate's evaluation: the calibrated estimate when the coefficients
270
- * describe this load, otherwise the computed floor where a floor can be
271
- * compared at all, otherwise `unknown`. Whatever kept the estimate from
272
- * forming stays in the reasons, so a floor verdict still says why it is only
273
- * a floor.
226
+ * One candidate's evaluation: the engine fitter's measurement when it ran,
227
+ * otherwise the computed floor where a floor can be compared at all, otherwise
228
+ * `unknown`. Whatever kept the measurement from forming stays in the reasons,
229
+ * so a floor verdict still says why it is only a floor.
274
230
  */
275
- function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode, floorApplies) {
231
+ function evaluate(candidate, resolveProfile, floorApplies, native, sole) {
232
+ const measured = nativeEvaluation(native);
233
+ if (measured && (measured.kind !== 'bare' || measured.verdict === 'does-not-fit' || sole)) {
234
+ return { candidate, result: measured };
235
+ }
276
236
  const profile = resolveProfile(candidate.model.sha256Checksum);
277
237
  if (!profile) {
278
238
  return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
@@ -284,46 +244,25 @@ function evaluate(candidate, platform, calibration, resources, resolveProfile, g
284
244
  result: unknown('an entry in `artifacts` has no resource profile in the catalog')
285
245
  };
286
246
  }
287
- const estimated = estimate(candidate, profile, extra, platform, calibration, resources, gpuMode);
288
- if (estimated.kind === 'estimate' || !floorApplies)
289
- return { candidate, result: estimated };
247
+ if (!floorApplies) {
248
+ return {
249
+ candidate,
250
+ result: unknown('no engine verdict, and the floor has no budget to be compared against')
251
+ };
252
+ }
290
253
  const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
291
254
  return {
292
255
  candidate,
293
256
  result: {
294
257
  kind: 'floor',
295
258
  bytes: floor.bytes,
296
- reasons: [...estimated.reasons, ...floor.reasons],
297
- assumptions: [...(estimated.assumptions ?? []), ...floor.assumptions]
259
+ reasons: floor.reasons,
260
+ assumptions: floor.assumptions
298
261
  }
299
262
  };
300
263
  }
301
264
  function unknown(reason) {
302
- return { kind: 'unknown', estimatorVersion: 'none', reasons: [reason] };
303
- }
304
- /** Runs the calibrated estimator for a candidate, or says why it cannot. */
305
- function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
306
- if (!calibration) {
307
- return unknown(platform
308
- ? `no validated calibration for ${platform}`
309
- : 'the runtime platform is not one this assessment covers');
310
- }
311
- // A GPU is present and neither GPU mode engaged, so the only coefficients
312
- // left were measured with the offload disabled — not how this host runs.
313
- if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
314
- return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
315
- }
316
- const estimator = ESTIMATORS[profile.engine];
317
- if (!estimator) {
318
- return unknown(`engine '${profile.engine}' has no estimator in this phase`);
319
- }
320
- return estimator({
321
- profile,
322
- workload: candidate.workload,
323
- extraArtifactBytes,
324
- calibration,
325
- hasGpu: hasGpu(resources)
326
- });
265
+ return { kind: 'unknown', reasons: [reason] };
327
266
  }
328
267
  /**
329
268
  * Sums the artifact bytes of companion constants.
@@ -343,24 +282,6 @@ function extraArtifactBytes(candidate, resolveProfile) {
343
282
  }
344
283
  return total;
345
284
  }
346
- /**
347
- * States when, and under what conditions, this platform's coefficients were
348
- * measured.
349
- *
350
- * The backend belongs in the result because the buffers these coefficients
351
- * cover are allocated by it, while the coefficients themselves are keyed by
352
- * platform alone — so a fixture measured on one backend is being applied to
353
- * every backend on that platform. That is a real caveat, and the caller is
354
- * entitled to see it rather than read the fixture.
355
- */
356
- function calibrationAssumption(platform, calibration) {
357
- const base = `${platform} coefficients were calibrated on ${calibration.measuredAt}`;
358
- const on = calibration.measuredOn;
359
- if (!on)
360
- return base;
361
- const device = on.device ? ` (${on.device})` : '';
362
- return `${base} against a ${on.backend} backend${device}, at ${on.kvElementBytes} bytes per KV element`;
363
- }
364
285
  /**
365
286
  * The GPUs the engine could actually execute on: `chooseBackend` passes over a
366
287
  * paravirtual adapter and a device with no graphics API this build talks to,
@@ -388,8 +309,8 @@ function tooSmallToHostAModel(gpu) {
388
309
  return (gpu.memoryTotalBytes.status === 'supported' && gpu.memoryTotalBytes.value < MIN_USABLE_GPU_BYTES);
389
310
  }
390
311
  // `gpuType.VIRTUAL`: the virtio / VMware / Hyper-V adapter a VM exposes. It
391
- // has no compute backend, so counting it as a GPU would deny every VM the
392
- // platform's own coefficients.
312
+ // has no compute backend, so counting it as a GPU would budget every VM
313
+ // against memory nothing can allocate.
393
314
  const GPU_TYPE_VIRTUAL = 3;
394
315
  function isVirtualDisplayAdapter(gpu) {
395
316
  return gpu.type.status === 'supported' && gpu.type.value === GPU_TYPE_VIRTUAL;
@@ -410,7 +331,7 @@ function allocatesFromSystemMemory(gpu) {
410
331
  * APU also exposes for its carve-out — a Ryzen 5000U reported `dedicated` with
411
332
  * over a gigabyte of "VRAM". Vulkan calls the same device INTEGRATED_GPU, but
412
333
  * that is not in the collector, so an AMD GPU on linux cannot be placed and
413
- * assesses as `unknown` until the engine's own device type is exposed.
334
+ * assesses as `unknown`.
414
335
  */
415
336
  function integratedIsIndistinguishable(gpu, platform) {
416
337
  if (!platform.startsWith('linux'))
@@ -418,8 +339,8 @@ function integratedIsIndistinguishable(gpu, platform) {
418
339
  return gpu.driverName.status === 'supported' && gpu.driverName.value === 'amdgpu';
419
340
  }
420
341
  // Ordered by the backends the addon actually builds, not by what the device's
421
- // drivers advertise: the NVIDIA calibration host advertises both CUDA and
422
- // Vulkan, and every load on it reports `ggml_vulkan`, never `ggml_cuda`. The
342
+ // drivers advertise: an NVIDIA host advertises both CUDA and Vulkan, and every
343
+ // load on it reports `ggml_vulkan`, never `ggml_cuda`. The
423
344
  // engine's own choice is not observable from here (`chooseBackend` is C++ and
424
345
  // only reaches the llama log), so this order has to track the addon's build.
425
346
  const GPU_BACKENDS = ['metal', 'vulkan', 'rocm', 'cuda', 'levelZero', 'opencl'];
@@ -451,9 +372,8 @@ function resolveGpuPlacement(resources, platform) {
451
372
  if (deviceBacked.length === 0) {
452
373
  const shared = gpus[0];
453
374
  const backend = backendOf(shared);
454
- // Two integrated devices on different backends would need two fixtures and
455
- // the engine picks one, so the same rule applies as for cards: no single
456
- // set of coefficients, no estimate. They share the budget either way.
375
+ // The engine picks one of them, so the same rule applies as for cards: no
376
+ // single backend, no placement. They share the budget either way.
457
377
  if (gpus.some((gpu) => backendOf(gpu) !== backend))
458
378
  return undefined;
459
379
  return {
@@ -489,8 +409,8 @@ function resolveGpuPlacement(resources, platform) {
489
409
  ...(gpu.name.status === 'supported' && { device: gpu.name.value })
490
410
  });
491
411
  }
492
- // One fixture covers one backend, and one basis: cards that disagree on
493
- // either cannot be assessed under a single set of coefficients.
412
+ // A placement names one backend and one basis, so cards that disagree on
413
+ // either name no device to budget against.
494
414
  const backend = targets[0].backend;
495
415
  if (targets.some((target) => target.backend !== backend))
496
416
  return undefined;
@@ -513,22 +433,31 @@ function gpuBudget(target, platform) {
513
433
  function tightest(budgets) {
514
434
  return budgets.reduce((least, budget) => budget.availableAfterReserveBytes < least.availableAfterReserveBytes ? budget : least);
515
435
  }
436
+ /**
437
+ * What the candidate budgets have to hold. `alsoBoundBy` is present only where
438
+ * they are a card's own memory, which host bytes never come out of; without it
439
+ * they are system memory and everything lands there.
440
+ */
441
+ function chargedToDevice(demand, alsoBoundBy) {
442
+ return alsoBoundBy ? demand.deviceBytes : totalBytes(demand);
443
+ }
516
444
  /**
517
445
  * `candidates` are alternatives — the engine pins the model to one of them, and
518
446
  * which one is not observable here — so a fit has to hold on the smallest and a
519
447
  * refusal on the largest. `alsoBoundBy` is a conjunction: a GPU load is paid
520
448
  * for in system RAM too, so both bounds apply.
521
449
  */
522
- function verdictAgainst(estimate, candidates, alsoBoundBy) {
450
+ function verdictAgainst(demand, candidates, alsoBoundBy) {
523
451
  const room = candidates.map((budget) => budget.availableAfterReserveBytes);
524
- const primary = estimate.upper <= Math.min(...room)
452
+ const onDevice = chargedToDevice(demand, alsoBoundBy);
453
+ const primary = onDevice <= Math.min(...room)
525
454
  ? 'likely-fits'
526
- : estimate.lower > Math.max(...room)
455
+ : onDevice > Math.max(...room)
527
456
  ? 'likely-too-large'
528
457
  : 'unknown';
529
458
  if (!alsoBoundBy)
530
459
  return primary;
531
- return worst(primary, compare(estimate, alsoBoundBy.availableAfterReserveBytes));
460
+ return worst(primary, totalBytes(demand) > alsoBoundBy.availableAfterReserveBytes ? 'likely-too-large' : 'likely-fits');
532
461
  }
533
462
  /** The more pessimistic of two verdicts. */
534
463
  function worst(a, b) {
@@ -629,31 +558,6 @@ function reserveBytes(availableBytes, platform) {
629
558
  const mobile = platform !== undefined && MOBILE_PLATFORMS.includes(platform);
630
559
  return Math.min(mobile ? 1 * GIB : 2 * GIB, Math.floor(availableBytes * 0.2));
631
560
  }
632
- /**
633
- * Combines per-model bounds under the declared execution mode.
634
- *
635
- * Every model is resident either way — what differs is the working peak:
636
- * `sequential` counts only the largest, `concurrent` counts them all.
637
- */
638
- function aggregate(estimates, execution) {
639
- let persistentLower = 0;
640
- let persistentUpper = 0;
641
- let workingLower = 0;
642
- let workingUpper = 0;
643
- for (const estimate of estimates) {
644
- persistentLower += estimate.persistent.lower;
645
- persistentUpper += estimate.persistent.upper;
646
- if (execution === 'concurrent') {
647
- workingLower += estimate.working.lower;
648
- workingUpper += estimate.working.upper;
649
- }
650
- else {
651
- workingLower = Math.max(workingLower, estimate.working.lower);
652
- workingUpper = Math.max(workingUpper, estimate.working.upper);
653
- }
654
- }
655
- return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
656
- }
657
561
  /**
658
562
  * The combined lower bound when at least one candidate has only a floor: every
659
563
  * estimate contributes its own lower bound, every floor its bytes, aggregated
@@ -664,40 +568,61 @@ function aggregate(estimates, execution) {
664
568
  function aggregateFloor(results, execution) {
665
569
  let persistent = 0;
666
570
  let working = 0;
571
+ let hostBytes = 0;
667
572
  for (const result of results) {
668
- if (result.kind === 'unknown')
573
+ if (result.kind === 'unknown' || result.kind === 'bare')
669
574
  return undefined;
670
575
  if (result.kind === 'floor') {
671
576
  persistent += result.bytes;
672
577
  continue;
673
578
  }
674
- persistent += result.persistent.lower;
579
+ persistent += result.persistent;
580
+ hostBytes += result.hostBytes;
581
+ working =
582
+ execution === 'concurrent' ? working + result.working : Math.max(working, result.working);
583
+ }
584
+ return { deviceBytes: persistent + working, hostBytes };
585
+ }
586
+ function totalBytes(demand) {
587
+ return demand.deviceBytes + demand.hostBytes;
588
+ }
589
+ /**
590
+ * The combined demand when every candidate was measured by its own engine.
591
+ * Each fitter answered about one model against the whole machine, so only the
592
+ * bytes compose: persistent is always resident, and the working peak is summed
593
+ * under `concurrent` and maximised under `sequential`.
594
+ */
595
+ function aggregateNative(natives, execution) {
596
+ let persistent = 0;
597
+ let working = 0;
598
+ let hostBytes = 0;
599
+ for (const native of natives) {
600
+ persistent += native.persistent;
601
+ hostBytes += native.hostBytes;
675
602
  working =
676
- execution === 'concurrent'
677
- ? working + result.working.lower
678
- : Math.max(working, result.working.lower);
603
+ execution === 'concurrent' ? working + native.working : Math.max(working, native.working);
679
604
  }
680
- return persistent + working;
605
+ return { deviceBytes: persistent + working, hostBytes };
681
606
  }
682
607
  /**
683
608
  * A floor can only refuse: over the largest candidate budget it is
684
609
  * `likely-too-large`, and anything else is `unknown`, because the cost above
685
610
  * the floor is unmeasured and could be anything.
686
611
  */
687
- function floorVerdict(floorBytes, candidates) {
612
+ function floorVerdict(floor, candidates, alsoBoundBy) {
688
613
  const room = candidates.map((budget) => budget.availableAfterReserveBytes);
689
- return floorBytes > Math.max(...room) ? 'likely-too-large' : 'unknown';
690
- }
691
- function compare(estimate, budget) {
692
- if (estimate.lower > budget)
614
+ if (chargedToDevice(floor, alsoBoundBy) > Math.max(...room))
615
+ return 'likely-too-large';
616
+ if (alsoBoundBy && totalBytes(floor) > alsoBoundBy.availableAfterReserveBytes) {
693
617
  return 'likely-too-large';
694
- if (estimate.upper <= budget)
695
- return 'likely-fits';
618
+ }
696
619
  return 'unknown';
697
620
  }
698
621
  function toModelResult(candidate, result, budget,
699
622
  /** Every candidate GPU budget, when the host has more than one. */
700
- deviceBudgets, alsoBoundBy) {
623
+ deviceBudgets,
624
+ /** The system budget a GPU load is also paid for out of. */
625
+ alsoBoundBy) {
701
626
  if (result.kind === 'unknown') {
702
627
  return {
703
628
  name: candidate.model.name,
@@ -707,7 +632,9 @@ deviceBudgets, alsoBoundBy) {
707
632
  };
708
633
  }
709
634
  if (result.kind === 'floor') {
710
- const verdict = budget ? floorVerdict(result.bytes, deviceBudgets ?? [budget]) : 'unknown';
635
+ const verdict = budget
636
+ ? floorVerdict({ deviceBytes: result.bytes, hostBytes: 0 }, deviceBudgets ?? [budget], alsoBoundBy)
637
+ : 'unknown';
711
638
  return {
712
639
  name: candidate.model.name,
713
640
  verdict,
@@ -721,23 +648,32 @@ deviceBudgets, alsoBoundBy) {
721
648
  ? 'no usable system-memory sample, so this model has no verdict'
722
649
  : verdict === 'likely-too-large'
723
650
  ? 'the computed floor alone exceeds the budget'
724
- : 'the computed floor is within the budget, but without calibration the cost above it is unbounded, so a fit cannot be claimed'
651
+ : 'the computed floor is within the budget, but the cost above it is unmeasured, so a fit cannot be claimed'
725
652
  ]
726
653
  };
727
654
  }
728
- const total = {
729
- lower: result.persistent.lower + result.working.lower,
730
- upper: result.persistent.upper + result.working.upper
731
- };
655
+ if (result.kind === 'bare') {
656
+ return {
657
+ name: candidate.model.name,
658
+ verdict: result.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
659
+ evidence: 'native-fit',
660
+ ...(candidate.device !== undefined && { device: candidate.device }),
661
+ estimatorVersion: result.estimatorVersion,
662
+ reasons: [...result.reasons]
663
+ };
664
+ }
665
+ const bytes = aggregateNative([result], 'sequential');
666
+ const verdict = result.verdict === 'does-not-fit'
667
+ ? 'likely-too-large'
668
+ : budget
669
+ ? verdictAgainst(bytes, deviceBudgets ?? [budget], alsoBoundBy)
670
+ : 'likely-fits';
732
671
  return {
733
672
  name: candidate.model.name,
734
- verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
735
- evidence: 'calibration',
673
+ verdict,
674
+ evidence: 'native-fit',
736
675
  ...(candidate.device !== undefined && { device: candidate.device }),
737
- estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
738
676
  estimatorVersion: result.estimatorVersion,
739
- reasons: budget
740
- ? [...result.reasons]
741
- : [...result.reasons, 'no usable system-memory sample, so this model has no verdict']
677
+ reasons: [...result.reasons]
742
678
  };
743
679
  }