@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -1,222 +1,77 @@
1
1
  import { getEngineLogger } from '../logging/index.js';
2
- import { classifyFit } from '../model-fit/classify.js';
3
- import { createFitRequest } from '../model-fit/request/create.js';
4
- import { runFit as runFitDefault } from '../model-fit/run/run.js';
5
- /**
6
- * The base every engine's fitter withholds by default. Made explicit here
7
- * because the resident-model reserve is added to it: setting a margin at all
8
- * replaces the engine default, so the base has to travel with it. It is also
9
- * the headroom `withMachineBudget` leaves on the system budget.
10
- */
11
- const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
12
- /**
13
- * Names the headroom policy: the fitter withholds
14
- * `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
15
- * in this worker, and a `fit` is then judged against the budget
16
- * `assessModelFit` reports, under that platform's basis and less the
17
- * `interactive-v1` reserve.
18
- */
19
- const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
20
- const BYTES_PER_MIB = 1024 * 1024;
21
- /**
22
- * Sums the on-disk weight sizes of every model currently registered in this
23
- * worker. The fallback for a host whose memory sample is unavailable, where
24
- * models this worker loaded are the one part of the machine's usage it can
25
- * still account for.
26
- *
27
- * Advisory and fail-open like everything else here: any failure to stat a file
28
- * contributes zero rather than an error.
29
- */
30
- async function defaultResidentModelBytes() {
31
- const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
32
- import('../runtime/model-registry.js'),
33
- import('bare-fs')
34
- ]);
35
- let bytes = 0;
36
- for (const id of getAllModelIds()) {
37
- const info = getModelInfo(id);
38
- if (info === null)
39
- continue;
40
- try {
41
- const stats = (await fsPromises.stat(info.path));
42
- bytes += stats.size;
43
- }
44
- catch {
45
- // Unreadable path: contribute nothing rather than fail the check.
46
- }
47
- }
48
- return bytes;
49
- }
50
- const PROVENANCE = {
51
- basis: 'native-probe',
52
- estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
53
- };
54
- function unknown(reason, message) {
55
- return message === undefined
56
- ? { ...PROVENANCE, verdict: 'unknown', reason }
57
- : { ...PROVENANCE, verdict: 'unknown', reason, message };
58
- }
59
- function mib(bytes) {
60
- return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
61
- }
62
- /**
63
- * What the operating system would give back, which no backend reports. Metal
64
- * answers with its working-set allowance minus this process's own allocations,
65
- * so another application's resident pages read as free, and the host figure
66
- * counts little beyond wired and compressed memory.
67
- */
68
- async function defaultAvailableSystemBytes() {
69
- const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
70
- import('../resources/instance.js'),
71
- import('../model-fit/assess.js'),
72
- import('../model-fit/platform.js')
73
- ]);
74
- const collector = getResourceCollector();
75
- if (!collector)
76
- return undefined;
77
- const platform = detectPlatform();
78
- const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
79
- const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
80
- if (budget === undefined)
81
- return undefined;
82
- return Math.max(0, budget.availableAfterReserveBytes);
83
- }
84
- /**
85
- * Whether the device's allocations come out of the pool the system budget
86
- * measures. Host bytes always do; device bytes do only where every allocation
87
- * is system RAM, which is what `boundBySystemMemory` decides.
88
- */
89
- async function deviceBytesCountAgainstSystem() {
90
- const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([import('../resources/instance.js'), import('../model-fit/assess.js'), import('../model-fit/platform.js')]);
91
- const collector = getResourceCollector();
92
- // Unknown counts them, which errs toward refusing.
93
- if (!collector)
94
- return true;
95
- const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
96
- return boundBySystemMemory(resources, detectPlatform());
97
- }
98
- /**
99
- * Peak the projection places on the pool the system budget measures. A
100
- * discrete card's memory is not that pool, so charging its bytes here would
101
- * refuse a load the machine can hold.
102
- */
103
- function projectedDemandBytes(outcome, countDevice) {
104
- const projection = outcome.projection;
105
- if (projection === undefined)
106
- return undefined;
107
- if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
108
- return undefined;
109
- return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
110
- }
111
- /**
112
- * Judges a `fit` against the machine. The fitter decides placement and reports
113
- * what the load costs; only this side knows what is free, so only this side can
114
- * refuse on capacity.
115
- *
116
- * The margin cannot carry this. It is subtracted from each device's own free,
117
- * and a device whose total sits below the machine's — Metal caps its allowance
118
- * near 74% of system memory — would be charged the shortfall twice.
119
- *
120
- * A projection with no byte totals leaves the engine's verdict standing.
121
- */
122
- function withMachineBudget(outcome, availableBytes, countDevice) {
123
- if (outcome.verdict !== 'fit' || availableBytes === undefined)
124
- return outcome;
125
- const demand = projectedDemandBytes(outcome, countDevice);
126
- if (demand === undefined)
127
- return outcome;
128
- if (demand <= availableBytes)
129
- return outcome;
130
- return {
131
- ...outcome,
132
- verdict: 'does-not-fit',
133
- reason: 'exceeds-available-memory',
134
- message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
135
- };
136
- }
137
- /** The device and host totals, for the engines that measure them. */
138
- function footprint(outcome) {
139
- const projection = outcome.projection;
140
- if (projection?.deviceBytes === undefined)
141
- return '';
142
- const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
143
- return ` — device ${mib(projection.deviceBytes)}${host}`;
2
+ import { fitLoad, unknownFit } from '../model-fit/fit-load.js';
3
+ import { describeAccounting, describePool } from '../model-fit/describe.js';
4
+ import { combineFits } from '../model-fit/sum.js';
5
+ /** The pools the load overflows, or the fitter's own refusal when none does. */
6
+ function refusalDetail(combined) {
7
+ const over = combined.pools.filter((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes);
8
+ if (over.length === 0)
9
+ return 'the engine fitter projects it will not fit its devices';
10
+ return `needs ${over.map(describePool).join(', ')}`;
144
11
  }
145
- function report(logger, input, outcome) {
12
+ function report(logger, input, fit, combined, detail) {
146
13
  const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
147
- if (outcome.verdict === 'fit') {
148
- const plan = outcome.plan;
149
- logger.info(`${prefix} projected to fit${plan === undefined
150
- ? footprint(outcome)
151
- : ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
14
+ const accounting = describeAccounting(combined.pools, [
15
+ { label: input.modelType, devices: fit.result.devices }
16
+ ]);
17
+ if (combined.verdict === 'fits') {
18
+ logger.info(`${prefix} projected to fit — ${accounting}`);
152
19
  return;
153
20
  }
154
- if (outcome.verdict === 'does-not-fit') {
155
- logger.warn(`${prefix} projected not to fit${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
21
+ if (combined.verdict === 'does-not-fit') {
22
+ logger.warn(`${prefix} projected not to fit: ${detail} — ${accounting}`);
156
23
  return;
157
24
  }
158
- // `unsupported-load` covers every load this check refuses up front — a model
159
- // type with no fitter, a companion set that did not resolve — so at `info` it
160
- // would tag those loads on every start. `debug` for those; `info` where a
161
- // fitter ran (or should have) and produced no verdict, because there "why was
162
- // there no evidence" is the most useful thing this check can report.
163
- const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
164
- if (outcome.reason === 'unsupported-load') {
25
+ const line = `${prefix} no fit verdict: ${fit.reason ?? 'a device reported no free memory'}`;
26
+ // A model type with no fitter answers this on every load, so it stays at
27
+ // `debug`; anywhere a fitter should have answered, `info` says why it did not.
28
+ if (fit.result.devices.length === 0 && fit.reason?.startsWith('no fitter') === true) {
165
29
  logger.debug(line);
166
30
  return;
167
31
  }
168
- logger.info(line);
32
+ logger.info(`${line} — ${accounting}`);
169
33
  }
170
34
  /**
171
- * Runs the advisory fit check for a load that is about to start, against the
172
- * fitter belonging to the engine that will run it.
35
+ * Checks a load that is about to start against what is free, with the fitter
36
+ * of the engine that will run it.
173
37
  *
174
- * Fail-open by construction: an unsupported shape, a fitter that crashed,
175
- * threw or did not return in time, and an unexpected internal error all
176
- * resolve to `unknown` and the caller proceeds with the ordinary load path.
177
- * This function never throws and never rejects.
38
+ * A fitter that rejects the load is logged and answers `unknown`, so the
39
+ * check never fails a load of its own accord. This function never throws.
178
40
  */
179
- export async function runAdvisoryFitCheck(input, options = {}) {
180
- let logger = options.logger;
41
+ export async function runLoadFitCheck(input, options = {}) {
42
+ const logger = options.logger ?? getEngineLogger();
43
+ let fit;
181
44
  try {
182
- logger ??= getEngineLogger();
183
- const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
184
- const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
185
- (options.extraResidentBytes ?? 0);
186
- const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
187
- // Always sent, even with a zero reserve, so the base margin has one source
188
- // and an engine default never decides it.
189
- const plan = createFitRequest({
190
- modelType: input.modelType,
45
+ fit = await fitLoad(input.modelType, {
191
46
  modelPath: input.modelPath,
192
47
  modelConfig: input.modelConfig,
193
- transcriptionWorkload: input.transcriptionWorkload,
194
- ttsWorkload: input.ttsWorkload,
195
48
  artifacts: input.artifacts,
196
- isShardedModel: input.isShardedModel,
197
- marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
49
+ isShardedModel: input.isShardedModel
50
+ }, {
51
+ ...(options.signal !== undefined && { signal: options.signal }),
52
+ ...(options.runFit !== undefined && { runFit: options.runFit })
198
53
  });
199
- if (!plan.supported) {
200
- const outcome = unknown('unsupported-load', plan.detail);
201
- report(logger, input, outcome);
202
- return outcome;
203
- }
204
- const result = await (options.runFit ?? runFitDefault)(plan.probe, options.signal !== undefined ? { signal: options.signal } : {});
205
- const outcome = result.status === 'completed'
206
- ? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
207
- : unknown(result.reason, result.message);
208
- report(logger, input, outcome);
209
- return outcome;
210
54
  }
211
55
  catch (error) {
212
- const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
56
+ const message = error instanceof Error ? error.message : String(error);
213
57
  try {
214
- if (logger !== undefined)
215
- report(logger, input, outcome);
58
+ logger.warn(`[advisory-fit:${input.modelType}:${input.modelId}] fit failed: ${message}`);
216
59
  }
217
60
  catch {
218
- // A failing logger must not turn an advisory check into a load failure.
61
+ // A failing logger must not turn the check into a load failure.
219
62
  }
220
- return outcome;
63
+ return { probe: { verdict: 'unknown', pools: [], models: [unknownFit(message).result] } };
221
64
  }
65
+ const combined = combineFits([fit.result]);
66
+ const detail = combined.verdict === 'does-not-fit' ? refusalDetail(combined) : undefined;
67
+ try {
68
+ report(logger, input, fit, combined, detail);
69
+ }
70
+ catch {
71
+ // A failing logger must not turn the check into a load failure.
72
+ }
73
+ return {
74
+ probe: { verdict: combined.verdict, pools: combined.pools, models: [fit.result] },
75
+ ...(detail !== undefined && { detail })
76
+ };
222
77
  }
@@ -1,72 +1,12 @@
1
- import type { AsrFitRequest, AsrFitResult } from '@qvac/asr-ggml';
2
- import type { AudiogenFitRequest, AudiogenFitResult } from '@qvac/audiogen-ggml';
3
- import type { BciFitRequest, BciFitResult } from '@qvac/bci-whispercpp';
4
- import type { DiffusionFitRequest, DiffusionFitResult, EsrganFitRequest, WorldFitRequest } from '@qvac/diffusion-cpp';
5
- import type { EmbedFitRequest, EmbedFitResult } from '@qvac/embed-llamacpp';
6
- import type LlmLlamacpp from '@qvac/llm-llamacpp';
7
- import type { OcrFitRequest, OcrFitResult } from '@qvac/ocr-ggml';
8
- import type TranslationNmtcpp from '@qvac/translation-nmtcpp';
9
- import type { TtsFitRequest, TtsFitResult } from '@qvac/tts-ggml';
10
- /**
11
- * The engine packages each expose their own `assessFit`, taking the same fields
12
- * their loader takes and returning the figures their fitter measures.
13
- */
14
- export type FitProbeRequest = {
15
- engine: 'llm-llamacpp';
16
- request: LlmLlamacpp.FitRequest;
17
- } | {
18
- engine: 'embed-llamacpp';
19
- request: EmbedFitRequest;
20
- } | {
21
- engine: 'asr-ggml';
22
- request: AsrFitRequest;
23
- } | {
24
- engine: 'bci-whispercpp';
25
- request: BciFitRequest;
26
- } | {
27
- engine: 'tts-ggml';
28
- request: TtsFitRequest;
29
- } | {
30
- engine: 'audiogen-ggml';
31
- request: AudiogenFitRequest;
32
- } | {
33
- engine: 'diffusion-cpp';
34
- request: DiffusionFitRequest | EsrganFitRequest | WorldFitRequest;
35
- } | {
36
- engine: 'ocr-ggml';
37
- request: OcrFitRequest;
38
- } | {
39
- engine: 'translation-nmtcpp';
40
- request: TranslationNmtcpp.FitRequest;
41
- };
42
- export type FitProbeEngine = FitProbeRequest['engine'];
43
- export type FitProbeResult = {
44
- engine: 'llm-llamacpp';
45
- result: LlmLlamacpp.FitResult;
46
- } | {
47
- engine: 'embed-llamacpp';
48
- result: EmbedFitResult;
49
- } | {
50
- engine: 'asr-ggml';
51
- result: AsrFitResult;
52
- } | {
53
- engine: 'bci-whispercpp';
54
- result: BciFitResult;
55
- } | {
56
- engine: 'tts-ggml';
57
- result: TtsFitResult;
58
- } | {
59
- engine: 'audiogen-ggml';
60
- result: AudiogenFitResult;
61
- } | {
62
- engine: 'diffusion-cpp';
63
- result: DiffusionFitResult;
64
- } | {
65
- engine: 'ocr-ggml';
66
- result: OcrFitResult;
67
- } | {
68
- engine: 'translation-nmtcpp';
69
- result: TranslationNmtcpp.FitResult;
70
- };
71
- /** Calls one engine's fitter, taken from the plugin that owns its addon. */
72
- export declare function callEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
1
+ import type { ModelFitDevice, ModelFitModelResult } from '../../schemas/assess-model-fit.js';
2
+ /** One device a load uses, as every addon's `assessFit` reports it. */
3
+ export type AddonFitDevice = ModelFitDevice;
4
+ /** The result every addon's `assessFit` returns. */
5
+ export type AddonFit = ModelFitModelResult;
6
+ /** A request for the fitter of the plugin that owns `addonPackage`. */
7
+ export interface FitProbeRequest {
8
+ addonPackage: string;
9
+ request: unknown;
10
+ }
11
+ /** Calls the fitter of a plugin that owns the request's addon. */
12
+ export declare function callEngineFit(probe: FitProbeRequest): Promise<AddonFit>;
@@ -1,13 +1,9 @@
1
- /** Calls one engine's fitter, taken from the plugin that owns its addon. */
1
+ /** Calls the fitter of a plugin that owns the request's addon. */
2
2
  export async function callEngineFit(probe) {
3
3
  const { getAllPlugins } = await import('../../plugins/registry.js');
4
- const addonPackage = `@qvac/${probe.engine}`;
5
- const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === addonPackage);
4
+ const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === probe.addonPackage && candidate.assessFit !== undefined);
6
5
  if (plugin?.assessFit === undefined) {
7
- throw new TypeError(`no registered plugin exposes assessFit for ${addonPackage}`);
6
+ throw new TypeError(`no registered plugin exposes assessFit for ${probe.addonPackage}`);
8
7
  }
9
- return {
10
- engine: probe.engine,
11
- result: await plugin.assessFit(probe.request)
12
- };
8
+ return plugin.assessFit(probe.request);
13
9
  }
@@ -1,14 +1,14 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
2
  import { callEngineFit } from '../../model-fit/run/engine.js';
3
- import type { FitProbeRequest, FitProbeResult } from '../../model-fit/run/engine.js';
3
+ import type { AddonFit, FitProbeRequest } from '../../model-fit/run/engine.js';
4
4
  /**
5
- * Why a run produced no projection. Every one of these is absence of evidence,
6
- * so the caller proceeds with the load it was going to run anyway.
5
+ * Why a run produced no fit. Each one makes the verdict `unknown`, so the
6
+ * caller proceeds with the load it was going to run anyway.
7
7
  */
8
- export type FitRunUnknownReason = 'cancelled' | 'crashed' | 'invocation-error' | 'timeout';
8
+ export type FitRunUnknownReason = 'cancelled' | 'crashed' | 'marker-error' | 'timeout';
9
9
  export type FitRunResult = {
10
10
  status: 'completed';
11
- probe: FitProbeResult;
11
+ fit: AddonFit;
12
12
  } | {
13
13
  status: 'unknown';
14
14
  reason: FitRunUnknownReason;
@@ -31,13 +31,16 @@ export declare function crashedMarkerPath(stateDir: string, probe: FitProbeReque
31
31
  * by its pid, so a marker whose process is gone is a fit that never finished,
32
32
  * while one whose process is alive is a fit still running there. The first
33
33
  * unfinished fit of a request leaves `.suspect` and the fit runs again. A
34
- * second one in a row leaves `.crashed`, and the request skips native until
35
- * that marker expires.
34
+ * second one in a row leaves `.crashed`, and the request skips the fitter
35
+ * until that marker expires.
36
36
  *
37
37
  * Fits of one request in this process share one marker, removed by the last.
38
38
  * An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
39
39
  * `timeout`, both at once, while the fitter runs on and releases the marker
40
40
  * when it returns. Until a timed-out fitter returns, further fits of its
41
41
  * request answer `timeout` without calling it again.
42
+ *
43
+ * A fitter that rejects, for a model it cannot read or a request it refuses,
44
+ * rejects this call with its error.
42
45
  */
43
46
  export declare function runFit(probe: FitProbeRequest, options?: RunFitOptions): Promise<FitRunResult>;
@@ -9,7 +9,7 @@ function unknown(reason, message) {
9
9
  function describe(error) {
10
10
  return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
11
11
  }
12
- /** How long a marker keeps a model off the native path. */
12
+ /** How long a marker keeps a request from calling its fitter. */
13
13
  const MARKER_TTL_MS = 7 * 24 * 60 * 60 * 1000;
14
14
  /** A fit sits in front of a load, so a fitter that never returns must not hold it. */
15
15
  const FIT_TIMEOUT_MS = 30_000;
@@ -17,13 +17,8 @@ const FIT_TIMEOUT_MS = 30_000;
17
17
  const inFlight = new Map();
18
18
  /** Requests whose fit timed out and has not returned. */
19
19
  const hung = new Set();
20
- /**
21
- * `marginBytes` is excluded: it carries the bytes of every model resident at
22
- * the time, so it differs between two attempts at the same load.
23
- */
24
20
  function markerKey(probe) {
25
- const { marginBytes: _margin, ...request } = probe.request;
26
- return generateShortHash(JSON.stringify({ engine: probe.engine, request }));
21
+ return generateShortHash(JSON.stringify(probe));
27
22
  }
28
23
  export function runningMarkerPath(stateDir, probe, pid = os.pid()) {
29
24
  return path.join(stateDir, `${markerKey(probe)}.${pid}.running`);
@@ -106,10 +101,7 @@ async function resolveStateDir(explicit) {
106
101
  async function settle(probe, callFit, stateDir) {
107
102
  const key = markerKey(probe);
108
103
  try {
109
- return { status: 'completed', probe: await callFit(probe) };
110
- }
111
- catch (error) {
112
- return unknown('invocation-error', describe(error));
104
+ return { status: 'completed', fit: await callFit(probe) };
113
105
  }
114
106
  finally {
115
107
  const remaining = (inFlight.get(key) ?? 1) - 1;
@@ -125,19 +117,26 @@ async function settle(probe, callFit, stateDir) {
125
117
  }
126
118
  }
127
119
  function bounded(fit, key, timeoutMs, signal) {
128
- return new Promise((resolve) => {
129
- const finish = (result) => {
120
+ return new Promise((resolve, reject) => {
121
+ const settled = () => {
130
122
  clearTimeout(timer);
131
123
  signal?.removeEventListener('abort', onAbort);
124
+ };
125
+ const finish = (result) => {
126
+ settled();
132
127
  resolve(result);
133
128
  };
129
+ const fail = (error) => {
130
+ settled();
131
+ reject(error);
132
+ };
134
133
  const onAbort = () => finish(unknown('cancelled', 'Fit was cancelled'));
135
134
  const timer = setTimeout(() => {
136
135
  hung.add(key);
137
136
  finish(unknown('timeout', `Fit did not return within ${timeoutMs} ms`));
138
137
  }, timeoutMs);
139
138
  signal?.addEventListener('abort', onAbort, { once: true });
140
- void fit.then(finish);
139
+ fit.then(finish, fail);
141
140
  });
142
141
  }
143
142
  /**
@@ -148,14 +147,17 @@ function bounded(fit, key, timeoutMs, signal) {
148
147
  * by its pid, so a marker whose process is gone is a fit that never finished,
149
148
  * while one whose process is alive is a fit still running there. The first
150
149
  * unfinished fit of a request leaves `.suspect` and the fit runs again. A
151
- * second one in a row leaves `.crashed`, and the request skips native until
152
- * that marker expires.
150
+ * second one in a row leaves `.crashed`, and the request skips the fitter
151
+ * until that marker expires.
153
152
  *
154
153
  * Fits of one request in this process share one marker, removed by the last.
155
154
  * An aborted `signal` answers `cancelled` and a fit past `timeoutMs` answers
156
155
  * `timeout`, both at once, while the fitter runs on and releases the marker
157
156
  * when it returns. Until a timed-out fitter returns, further fits of its
158
157
  * request answer `timeout` without calling it again.
158
+ *
159
+ * A fitter that rejects, for a model it cannot read or a request it refuses,
160
+ * rejects this call with its error.
159
161
  */
160
162
  export async function runFit(probe, options = {}) {
161
163
  if (options.signal?.aborted === true) {
@@ -177,7 +179,9 @@ export async function runFit(probe, options = {}) {
177
179
  const unfinished = unfinishedFits(stateDir, probe);
178
180
  if (exists(crashed) || (unfinished.length > 0 && exists(suspect))) {
179
181
  try {
180
- mark(crashed);
182
+ // Rewriting an existing marker would restart its TTL on every load.
183
+ if (!exists(crashed))
184
+ mark(crashed);
181
185
  remove(suspect);
182
186
  unfinished.forEach(remove);
183
187
  }
@@ -195,7 +199,7 @@ export async function runFit(probe, options = {}) {
195
199
  mark(runningMarkerPath(stateDir, probe));
196
200
  }
197
201
  catch (error) {
198
- return unknown('invocation-error', `Fit marker could not be written: ${describe(error)}`);
202
+ return unknown('marker-error', `Fit marker could not be written: ${describe(error)}`);
199
203
  }
200
204
  }
201
205
  inFlight.set(key, held + 1);
@@ -0,0 +1,32 @@
1
+ import type { AddonFitDevice } from '../model-fit/run/engine.js';
2
+ import type { ModelFitPool, ModelFitVerdict } from '../schemas/assess-model-fit.js';
3
+ /**
4
+ * The share of a pool's free memory a set of models may take. The rest absorbs
5
+ * what the machine allocates between the check and the loads.
6
+ */
7
+ export declare const USABLE_SHARE = 0.9;
8
+ /** The pool of host memory: the host CPU's row and every GPU that shares it. */
9
+ export declare const HOST_POOL = "host";
10
+ export interface ModelAnswer {
11
+ status: ModelFitVerdict;
12
+ devices: readonly AddonFitDevice[];
13
+ }
14
+ export interface CombinedFit {
15
+ verdict: ModelFitVerdict;
16
+ pools: ModelFitPool[];
17
+ }
18
+ /**
19
+ * Sums what a set of models takes on each memory pool and judges the set
20
+ * against what is free.
21
+ *
22
+ * A device joins the pool of its `id`, or the host pool when `unified`. A
23
+ * pool's free and total memory are the lowest figures reported for it.
24
+ *
25
+ * A single model's verdict is its addon's own, which applies the engine's
26
+ * margin. A set does not fit when any pool takes more than
27
+ * `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
28
+ * known so far is a lower bound, so either holds whatever the unknown models
29
+ * add. Otherwise the set is `unknown` when any model is unknown or any pool has
30
+ * no free figure.
31
+ */
32
+ export declare function combineFits(models: readonly ModelAnswer[]): CombinedFit;
@@ -0,0 +1,60 @@
1
+ /**
2
+ * The share of a pool's free memory a set of models may take. The rest absorbs
3
+ * what the machine allocates between the check and the loads.
4
+ */
5
+ export const USABLE_SHARE = 0.9;
6
+ /** The pool of host memory: the host CPU's row and every GPU that shares it. */
7
+ export const HOST_POOL = 'host';
8
+ function lowest(figures) {
9
+ return figures.length > 0 ? Math.min(...figures) : undefined;
10
+ }
11
+ /**
12
+ * Sums what a set of models takes on each memory pool and judges the set
13
+ * against what is free.
14
+ *
15
+ * A device joins the pool of its `id`, or the host pool when `unified`. A
16
+ * pool's free and total memory are the lowest figures reported for it.
17
+ *
18
+ * A single model's verdict is its addon's own, which applies the engine's
19
+ * margin. A set does not fit when any pool takes more than
20
+ * `USABLE_SHARE` of its free memory or any model does not fit alone. Usage
21
+ * known so far is a lower bound, so either holds whatever the unknown models
22
+ * add. Otherwise the set is `unknown` when any model is unknown or any pool has
23
+ * no free figure.
24
+ */
25
+ export function combineFits(models) {
26
+ const totals = new Map();
27
+ for (const model of models) {
28
+ for (const device of model.devices) {
29
+ const key = device.unified ? HOST_POOL : device.id;
30
+ const pool = totals.get(key) ?? { usedBytes: 0, free: [], total: [] };
31
+ pool.usedBytes += device.usedBytes;
32
+ if (device.freeBytes !== undefined)
33
+ pool.free.push(device.freeBytes);
34
+ if (device.totalBytes !== undefined)
35
+ pool.total.push(device.totalBytes);
36
+ totals.set(key, pool);
37
+ }
38
+ }
39
+ const pools = [...totals].map(([id, { usedBytes, free, total }]) => {
40
+ const freeBytes = lowest(free);
41
+ const totalBytes = lowest(total);
42
+ return {
43
+ id,
44
+ unified: id === HOST_POOL,
45
+ usedBytes,
46
+ ...(freeBytes !== undefined && { freeBytes }),
47
+ ...(totalBytes !== undefined && { totalBytes })
48
+ };
49
+ });
50
+ const [only] = models;
51
+ if (models.length === 1 && only !== undefined)
52
+ return { verdict: only.status, pools };
53
+ const over = pools.some((pool) => pool.freeBytes !== undefined && pool.usedBytes > pool.freeBytes * USABLE_SHARE);
54
+ if (over || models.some((model) => model.status === 'does-not-fit')) {
55
+ return { verdict: 'does-not-fit', pools };
56
+ }
57
+ const unmeasured = models.some((model) => model.status === 'unknown') ||
58
+ pools.some((pool) => pool.freeBytes === undefined);
59
+ return { verdict: unmeasured ? 'unknown' : 'fits', pools };
60
+ }