@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -1,27 +0,0 @@
1
- import { type LoadDescription } from './resolve-load-from-stubs.js';
2
- import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
3
- import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
4
- export type LoadFitOutcome = {
5
- status: 'projected';
6
- fit: AdvisoryFitOutcome;
7
- } | {
8
- status: 'no-stub';
9
- reason: FitStubUnavailableReason;
10
- message?: string;
11
- } | {
12
- status: 'unsupported-load';
13
- detail: string;
14
- };
15
- export interface LoadFitOptions {
16
- stub?: FitStubOptions;
17
- fit?: AdvisoryFitOptions;
18
- }
19
- /**
20
- * Projects the load a caller describes, before its weights exist locally.
21
- *
22
- * The same probe `loadModel` runs, over the same config and artifacts that
23
- * load's own plugin resolves, pointed at files that are tens of KB instead of
24
- * gigabytes. It inherits the probe's fail-open contract: every failure is an
25
- * `unknown` verdict or a non-projected outcome, never a throw.
26
- */
27
- export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
@@ -1,33 +0,0 @@
1
- import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
2
- import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
3
- /**
4
- * Projects the load a caller describes, before its weights exist locally.
5
- *
6
- * The same probe `loadModel` runs, over the same config and artifacts that
7
- * load's own plugin resolves, pointed at files that are tens of KB instead of
8
- * gigabytes. It inherits the probe's fail-open contract: every failure is an
9
- * `unknown` verdict or a non-projected outcome, never a throw.
10
- */
11
- export async function projectFitFromLoad(load, modelId, options = {}) {
12
- const resolved = await resolveLoadFromStubs(load, options.stub);
13
- if (resolved.status !== 'resolved')
14
- return resolved;
15
- try {
16
- const fit = await runAdvisoryFitCheck({
17
- modelId,
18
- modelType: load.modelType,
19
- modelPath: resolved.modelPath,
20
- modelConfig: resolved.modelConfig,
21
- transcriptionWorkload: load.transcriptionWorkload,
22
- ttsWorkload: load.ttsWorkload,
23
- artifacts: resolved.artifacts,
24
- // Nothing here assembles a shard set, so no load may claim the fitter
25
- // is looking at a split model.
26
- isShardedModel: false
27
- }, options.fit);
28
- return { status: 'projected', fit };
29
- }
30
- finally {
31
- await resolved.release();
32
- }
33
- }
@@ -1,31 +0,0 @@
1
- import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
2
- import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
3
- export declare const FLOOR_VERSION = "floor-v1";
4
- export interface FloorInput {
5
- profile: ModelResourceProfile;
6
- workload: ModelFitWorkload;
7
- /** Artifact bytes of companion constants the caller passed alongside. */
8
- extraArtifactBytes: number;
9
- }
10
- /** A lower bound with no upper bound, and how it was assembled. */
11
- export interface FloorResult {
12
- bytes: number;
13
- reasons: readonly string[];
14
- assumptions: readonly string[];
15
- }
16
- /**
17
- * The smallest resident footprint the catalog facts prove: artifact bytes, plus
18
- * the KV cache for a llama.cpp model at the cheapest element width the engine
19
- * can default to.
20
- *
21
- * Every term the real load adds on top — weight slack, engine overhead, compute
22
- * buffers, a completion's working peak — is non-negative, so a floor with all of
23
- * them at zero is unconditionally valid on any platform and any backend. It can
24
- * therefore refuse a model (`likely-too-large` when even the floor is over
25
- * budget) but never confirm one, since how far above the floor the real load
26
- * lands is unmeasured here.
27
- *
28
- * This is the zero-fetch gate. It reads nothing but the catalog and stays the
29
- * fallback wherever no engine measurement is available.
30
- */
31
- export declare function computeFloor(input: FloorInput): FloorResult;
@@ -1,65 +0,0 @@
1
- import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from './kv-cache.js';
2
- export const FLOOR_VERSION = 'floor-v1';
3
- /**
4
- * Engines whose load builds a KV cache the file sizes exactly, so the floor can
5
- * count it. Every other engine's working memory is engine-owned and unsized
6
- * here; the floor for those is the weights alone.
7
- */
8
- const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
9
- /**
10
- * The smallest resident footprint the catalog facts prove: artifact bytes, plus
11
- * the KV cache for a llama.cpp model at the cheapest element width the engine
12
- * can default to.
13
- *
14
- * Every term the real load adds on top — weight slack, engine overhead, compute
15
- * buffers, a completion's working peak — is non-negative, so a floor with all of
16
- * them at zero is unconditionally valid on any platform and any backend. It can
17
- * therefore refuse a model (`likely-too-large` when even the floor is over
18
- * budget) but never confirm one, since how far above the floor the real load
19
- * lands is unmeasured here.
20
- *
21
- * This is the zero-fetch gate. It reads nothing but the catalog and stays the
22
- * fallback wherever no engine measurement is available.
23
- */
24
- export function computeFloor(input) {
25
- const { profile, workload, extraArtifactBytes } = input;
26
- const reasons = [];
27
- const assumptions = [
28
- KV_CACHE_ENGINES.has(profile.engine)
29
- ? LLAMA_WEIGHTS_ASSUMPTION
30
- : 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
31
- 'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
32
- ];
33
- const artifactBytes = profile.artifactBytes + extraArtifactBytes;
34
- if (extraArtifactBytes > 0) {
35
- assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
36
- }
37
- if (!KV_CACHE_ENGINES.has(profile.engine)) {
38
- reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
39
- return { bytes: artifactBytes, reasons, assumptions };
40
- }
41
- if (workload.kind !== 'llm') {
42
- reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
43
- return { bytes: artifactBytes, reasons, assumptions };
44
- }
45
- const facts = profile.ggufFacts;
46
- if (!facts) {
47
- reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
48
- return { bytes: artifactBytes, reasons, assumptions };
49
- }
50
- let contextTokens = workload.contextTokens ?? facts.contextLength;
51
- if (contextTokens > facts.contextLength) {
52
- assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
53
- contextTokens = facts.contextLength;
54
- }
55
- // The narrowest default the engine can pick on any backend, so the floor
56
- // holds whether the model lands on a GPU (q8_0 with flash attention) or on
57
- // the CPU (f16). Architectures that disable flash attention stay f16 anyway.
58
- const element = narrowestKvElementBytes(facts);
59
- const kv = kvCacheFloorBytes(facts, contextTokens, element);
60
- assumptions.push(element < 2
61
- ? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
62
- : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
63
- reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
64
- return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
65
- }
@@ -1,23 +0,0 @@
1
- import type { GgufFacts } from '../../schemas/model-resource-profile.js';
2
- export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
3
- /**
4
- * The narrowest KV-cache element width the engine can default to for this
5
- * model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
6
- */
7
- export declare function narrowestKvElementBytes(facts: GgufFacts): number;
8
- /**
9
- * The smallest KV cache the requested context can need, at a fixed element
10
- * width and clamped to the trained context.
11
- *
12
- * Three cases, in order of how much the file actually tells us:
13
- *
14
- * 1. **Per-layer classes** — the file describes attention per block, so the
15
- * cache is summed exactly, with sliding-window blocks capped at their window.
16
- * 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
17
- * blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
18
- * blocks are which is engine-owned, so the fewest full blocks are counted.
19
- * 3. **Flat** — every block holds the same cache. A declared sliding window
20
- * with no per-layer pattern is engine-owned, so every block is counted
21
- * windowed.
22
- */
23
- export declare function kvCacheFloorBytes(facts: GgufFacts, contextTokens: number, elementBytes: number): number;
@@ -1,85 +0,0 @@
1
- /**
2
- * KV-cache element widths, in bytes per element.
3
- *
4
- * `f16` is the CPU default. On a Metal/Vulkan GPU backend with flash attention
5
- * on — the SDK's own defaults — `llm-llamacpp` defaults the cache to `q8_0`
6
- * instead (`addon/src/model-interface/LoadFitNormalization.cpp`, QVAC-21318).
7
- * `q8_0` packs 32 elements into a 34-byte block.
8
- */
9
- const F16_BYTES_PER_ELEMENT = 2;
10
- const Q8_0_BYTES_PER_ELEMENT = 34 / 32;
11
- /** Recurrent/SSM state is kept in f32. */
12
- const SSM_STATE_BYTES_PER_ELEMENT = 4;
13
- /**
14
- * Architectures that disable flash attention, so the GPU `q8_0` KV default
15
- * never applies and the cache stays `f16` on every backend.
16
- */
17
- function disablesFlashAttention(architecture) {
18
- return architecture.startsWith('bitnet');
19
- }
20
- // Shared with the computed floor so a mixed set carries one weights assumption, not two.
21
- export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
22
- /**
23
- * The narrowest KV-cache element width the engine can default to for this
24
- * model, in bytes: q8_0 where flash attention allows it, f16 otherwise.
25
- */
26
- export function narrowestKvElementBytes(facts) {
27
- return disablesFlashAttention(facts.architecture) ? F16_BYTES_PER_ELEMENT : Q8_0_BYTES_PER_ELEMENT;
28
- }
29
- /**
30
- * The smallest KV cache the requested context can need, at a fixed element
31
- * width and clamped to the trained context.
32
- *
33
- * Three cases, in order of how much the file actually tells us:
34
- *
35
- * 1. **Per-layer classes** — the file describes attention per block, so the
36
- * cache is summed exactly, with sliding-window blocks capped at their window.
37
- * 2. **Hybrid attention/recurrent** — `full_attention_interval` says how many
38
- * blocks hold a cache at all; the rest hold a fixed-size SSM state. Which
39
- * blocks are which is engine-owned, so the fewest full blocks are counted.
40
- * 3. **Flat** — every block holds the same cache. A declared sliding window
41
- * with no per-layer pattern is engine-owned, so every block is counted
42
- * windowed.
43
- */
44
- export function kvCacheFloorBytes(facts, contextTokens, elementBytes) {
45
- const tokens = Math.min(contextTokens, facts.contextLength);
46
- if (facts.kvLayerClasses && facts.kvLayerClasses.length > 0) {
47
- return layerClassBytes(facts.kvLayerClasses, facts, tokens, elementBytes);
48
- }
49
- const perBlockPerToken = facts.headCountKv * (facts.keyLength + facts.valueLength);
50
- if (facts.fullAttentionInterval && facts.fullAttentionInterval > 1) {
51
- const fullBlocks = Math.floor(facts.blockCount / facts.fullAttentionInterval);
52
- const ssm = ssmStateBytes(facts, facts.blockCount - fullBlocks);
53
- return fullBlocks * perBlockPerToken * tokens * elementBytes + ssm;
54
- }
55
- if (facts.slidingWindow) {
56
- const windowedTokens = Math.min(tokens, facts.slidingWindow);
57
- return facts.blockCount * perBlockPerToken * windowedTokens * elementBytes;
58
- }
59
- return facts.blockCount * perBlockPerToken * tokens * elementBytes;
60
- }
61
- function layerClassBytes(classes, facts, contextTokens, elementBytes) {
62
- let total = 0;
63
- for (const layerClass of classes) {
64
- const tokens = layerClass.windowed && facts.slidingWindow
65
- ? Math.min(contextTokens, facts.slidingWindow)
66
- : contextTokens;
67
- total +=
68
- layerClass.count *
69
- layerClass.headCountKv *
70
- (layerClass.keyLength + layerClass.valueLength) *
71
- tokens *
72
- elementBytes;
73
- }
74
- return total;
75
- }
76
- /**
77
- * Fixed recurrent state for the blocks of a hybrid model that hold no KV cache:
78
- * the SSM state plus its convolution window, per block.
79
- */
80
- function ssmStateBytes(facts, recurrentBlocks) {
81
- if (!facts.ssmInnerSize || !facts.ssmStateSize || recurrentBlocks <= 0)
82
- return 0;
83
- const perBlock = facts.ssmInnerSize * facts.ssmStateSize + facts.ssmInnerSize * (facts.ssmConvKernel ?? 0);
84
- return recurrentBlocks * perBlock * SSM_STATE_BYTES_PER_ELEMENT;
85
- }
@@ -1,54 +0,0 @@
1
- import type { AbortSignal } from 'bare-abort-controller';
2
- import type { Logger } from '../../../logging/types.js';
3
- import type { TtsFitWorkload } from '../../../schemas/assess-model-fit.js';
4
- import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict, TranscriptionFitWorkload } from '../../../schemas/index.js';
5
- import { runFit as runFitDefault } from './run-fit.js';
6
- /**
7
- * `fit` and `does-not-fit` are projections of the load the SDK is about to run.
8
- * The real loader neither consumes nor verifies the fitted plan.
9
- *
10
- * The outcome is the wire shape: `loadModel` hands it to the model registry and
11
- * `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
12
- * log line.
13
- */
14
- export type AdvisoryFitVerdict = NativeProbeVerdict;
15
- export type AdvisoryFitOutcome = NativeProbeFit;
16
- export interface AdvisoryFitInput {
17
- transcriptionWorkload?: TranscriptionFitWorkload | undefined;
18
- ttsWorkload?: TtsFitWorkload | undefined;
19
- modelId: string;
20
- modelType: CanonicalModelType;
21
- modelPath: string;
22
- modelConfig: unknown;
23
- artifacts?: Record<string, string> | undefined;
24
- isShardedModel: boolean;
25
- }
26
- /**
27
- * Injection seams. Every field defaults to the real runtime dependency; tests
28
- * substitute them rather than mocking modules.
29
- */
30
- export interface AdvisoryFitOptions {
31
- signal?: AbortSignal;
32
- mobile?: boolean;
33
- timeoutMs?: number;
34
- runFit?: typeof runFitDefault;
35
- logger?: Logger;
36
- availableSystemBytes?: () => Promise<number | undefined>;
37
- countsDeviceRows?: () => Promise<boolean>;
38
- residentModelBytes?: () => Promise<number>;
39
- /**
40
- * Bytes to hold back beyond what is resident, for models the caller counts
41
- * as loaded alongside this one but that this worker does not yet hold.
42
- */
43
- extraResidentBytes?: number;
44
- }
45
- /**
46
- * Runs the advisory fit check for a load that is about to start, against the
47
- * fitter belonging to the engine that will run it.
48
- *
49
- * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
50
- * malformed response, and an unexpected internal error all resolve to `unknown`
51
- * and the caller proceeds with the ordinary load path. This function never
52
- * throws and never rejects.
53
- */
54
- export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
@@ -1,248 +0,0 @@
1
- import { getEngineLogger } from '../../../logging/index.js';
2
- import { classifyFit } from './classify-fit.js';
3
- import { createFitRequest } from './create-fit-request.js';
4
- import { runFit as runFitDefault } from './run-fit.js';
5
- /**
6
- * Shorter than the supervisor's own 60s default: this check sits in front of a
7
- * real load, so a wedged child must not hold the load for a full minute. Every
8
- * expiry is `unknown` and the load continues.
9
- */
10
- const ADVISORY_FIT_TIMEOUT_MS = 30_000;
11
- /**
12
- * The base every engine's fitter withholds by default. Made explicit here
13
- * because the resident-model reserve is added to it: setting a margin at all
14
- * replaces the engine default, so the base has to travel with it. It is also
15
- * the headroom `withMachineBudget` leaves on the system budget.
16
- */
17
- const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
18
- /**
19
- * Names the headroom policy: the fitter withholds
20
- * `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
21
- * in this worker, and a `fit` is then judged against the budget
22
- * `assessModelFit` reports, under that platform's basis and less the
23
- * `interactive-v1` reserve.
24
- */
25
- const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
26
- const BYTES_PER_MIB = 1024 * 1024;
27
- /**
28
- * The mobile runtime flag is imported lazily: that module reaches Bare-only
29
- * bindings, and resolving it eagerly would make this orchestration untestable
30
- * outside a Bare runtime.
31
- */
32
- async function resolveMobile(explicit) {
33
- if (explicit !== undefined)
34
- return explicit;
35
- const { isMobile } = await import('../../../runtime/state.js');
36
- return isMobile();
37
- }
38
- /**
39
- * Sums the on-disk weight sizes of every model currently registered in this
40
- * worker. The fallback for a host whose memory sample is unavailable, where
41
- * models this worker loaded are the one part of the machine's usage it can
42
- * still account for.
43
- *
44
- * Advisory and fail-open like everything else here: any failure to stat a file
45
- * contributes zero rather than an error.
46
- */
47
- async function defaultResidentModelBytes() {
48
- const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
49
- import('../../../runtime/model-registry.js'),
50
- import('bare-fs')
51
- ]);
52
- let bytes = 0;
53
- for (const id of getAllModelIds()) {
54
- const info = getModelInfo(id);
55
- if (info === null)
56
- continue;
57
- try {
58
- const stats = (await fsPromises.stat(info.path));
59
- bytes += stats.size;
60
- }
61
- catch {
62
- // Unreadable path: contribute nothing rather than fail the check.
63
- }
64
- }
65
- return bytes;
66
- }
67
- const PROVENANCE = {
68
- basis: 'native-probe',
69
- estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
70
- };
71
- function unknown(reason, message) {
72
- return message === undefined
73
- ? { ...PROVENANCE, verdict: 'unknown', reason }
74
- : { ...PROVENANCE, verdict: 'unknown', reason, message };
75
- }
76
- function mib(bytes) {
77
- return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
78
- }
79
- /**
80
- * What the operating system would give back, which no backend reports. Metal
81
- * answers with its working-set allowance minus this process's own allocations,
82
- * so another application's resident pages read as free, and the host figure
83
- * counts little beyond wired and compressed memory.
84
- */
85
- async function defaultAvailableSystemBytes() {
86
- const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
87
- import('../../instance.js'),
88
- import('../assess.js'),
89
- import('../platform.js')
90
- ]);
91
- const collector = getResourceCollector();
92
- if (!collector)
93
- return undefined;
94
- const platform = detectPlatform();
95
- const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
96
- const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
97
- if (budget === undefined)
98
- return undefined;
99
- return Math.max(0, budget.availableAfterReserveBytes);
100
- }
101
- /**
102
- * Whether the device's allocations come out of the pool the system budget
103
- * measures. Host bytes always do; device bytes do only where every allocation
104
- * is system RAM, which is what `boundBySystemMemory` decides.
105
- */
106
- async function deviceBytesCountAgainstSystem() {
107
- const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
108
- import('../../instance.js'),
109
- import('../assess.js'),
110
- import('../platform.js')
111
- ]);
112
- const collector = getResourceCollector();
113
- // Unknown counts them, which errs toward refusing.
114
- if (!collector)
115
- return true;
116
- const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
117
- return boundBySystemMemory(resources, detectPlatform());
118
- }
119
- /**
120
- * Peak the projection places on the pool the system budget measures. A
121
- * discrete card's memory is not that pool, so charging its bytes here would
122
- * refuse a load the machine can hold.
123
- */
124
- function projectedDemandBytes(outcome, countDevice) {
125
- const projection = outcome.projection;
126
- if (projection === undefined)
127
- return undefined;
128
- if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
129
- return undefined;
130
- return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
131
- }
132
- /**
133
- * Judges a `fit` against the machine. The fitter decides placement and reports
134
- * what the load costs; only this side knows what is free, so only this side can
135
- * refuse on capacity.
136
- *
137
- * The margin cannot carry this. It is subtracted from each device's own free,
138
- * and a device whose total sits below the machine's — Metal caps its allowance
139
- * near 74% of system memory — would be charged the shortfall twice.
140
- *
141
- * A projection with no byte totals leaves the engine's verdict standing.
142
- */
143
- function withMachineBudget(outcome, availableBytes, countDevice) {
144
- if (outcome.verdict !== 'fit' || availableBytes === undefined)
145
- return outcome;
146
- const demand = projectedDemandBytes(outcome, countDevice);
147
- if (demand === undefined)
148
- return outcome;
149
- if (demand <= availableBytes)
150
- return outcome;
151
- return {
152
- ...outcome,
153
- verdict: 'does-not-fit',
154
- reason: 'exceeds-available-memory',
155
- message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
156
- };
157
- }
158
- /** The device and host totals, for the engines that measure them. */
159
- function footprint(outcome) {
160
- const projection = outcome.projection;
161
- if (projection?.deviceBytes === undefined)
162
- return '';
163
- const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
164
- return ` — device ${mib(projection.deviceBytes)}${host}`;
165
- }
166
- function report(logger, input, outcome) {
167
- const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
168
- if (outcome.verdict === 'fit') {
169
- const plan = outcome.plan;
170
- logger.info(`${prefix} projected to fit${plan === undefined
171
- ? footprint(outcome)
172
- : ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
173
- return;
174
- }
175
- if (outcome.verdict === 'does-not-fit') {
176
- logger.warn(`${prefix} projected not to fit${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
177
- return;
178
- }
179
- // `unsupported-load` covers every load this check refuses up front — a model
180
- // type with no fitter, a companion set that did not resolve — so at `info` it
181
- // would tag those loads on every start. `debug` for those; `info` where a
182
- // fitter ran (or should have) and produced no verdict, because there "why was
183
- // there no evidence" is the most useful thing this check can report.
184
- const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
185
- if (outcome.reason === 'unsupported-load') {
186
- logger.debug(line);
187
- return;
188
- }
189
- logger.info(line);
190
- }
191
- /**
192
- * Runs the advisory fit check for a load that is about to start, against the
193
- * fitter belonging to the engine that will run it.
194
- *
195
- * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
196
- * malformed response, and an unexpected internal error all resolve to `unknown`
197
- * and the caller proceeds with the ordinary load path. This function never
198
- * throws and never rejects.
199
- */
200
- export async function runAdvisoryFitCheck(input, options = {}) {
201
- let logger = options.logger;
202
- try {
203
- logger ??= getEngineLogger();
204
- const mobile = await resolveMobile(options.mobile);
205
- const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
206
- const residentBytes = (await (options.residentModelBytes ?? defaultResidentModelBytes)()) +
207
- (options.extraResidentBytes ?? 0);
208
- const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
209
- // Always sent, even with a zero reserve, so the base margin has one source
210
- // and an engine default never decides it.
211
- const plan = createFitRequest({
212
- modelType: input.modelType,
213
- modelPath: input.modelPath,
214
- modelConfig: input.modelConfig,
215
- transcriptionWorkload: input.transcriptionWorkload,
216
- ttsWorkload: input.ttsWorkload,
217
- artifacts: input.artifacts,
218
- isShardedModel: input.isShardedModel,
219
- marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
220
- });
221
- if (!plan.supported) {
222
- const outcome = unknown('unsupported-load', plan.detail);
223
- report(logger, input, outcome);
224
- return outcome;
225
- }
226
- const result = await (options.runFit ?? runFitDefault)(plan.probe, {
227
- mobile,
228
- timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
229
- ...(options.signal !== undefined && { signal: options.signal })
230
- });
231
- const outcome = result.status === 'completed'
232
- ? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
233
- : unknown(result.reason, result.message);
234
- report(logger, input, outcome);
235
- return outcome;
236
- }
237
- catch (error) {
238
- const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
239
- try {
240
- if (logger !== undefined)
241
- report(logger, input, outcome);
242
- }
243
- catch {
244
- // A failing logger must not turn an advisory check into a load failure.
245
- }
246
- return outcome;
247
- }
248
- }
@@ -1,2 +0,0 @@
1
- import type { FitProbeRequest, FitProbeResult } from './engine-fit.js';
2
- export declare function callChildEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
@@ -1,49 +0,0 @@
1
- /**
2
- * Loads the one addon a probe names and asks it for a projection. The child
3
- * registers no plugins, so the fitter cannot come off the registry here. Only
4
- * the child reaches this module, and the supervisor spawns it by path rather
5
- * than importing it, so these specifiers stay out of the graph a host bundles.
6
- */
7
- async function assessFitOf(probe) {
8
- switch (probe.engine) {
9
- case 'llm-llamacpp': {
10
- const { default: LlmLlamacpp } = await import('@qvac/llm-llamacpp');
11
- return LlmLlamacpp.assessFit(probe.request);
12
- }
13
- case 'embed-llamacpp': {
14
- const { assessFit } = await import('@qvac/embed-llamacpp');
15
- return assessFit(probe.request);
16
- }
17
- case 'asr-ggml': {
18
- const { default: ASRGgml } = await import('@qvac/asr-ggml');
19
- return ASRGgml.assessFit(probe.request);
20
- }
21
- case 'bci-whispercpp': {
22
- const { assessFit } = await import('@qvac/bci-whispercpp');
23
- return assessFit(probe.request);
24
- }
25
- case 'tts-ggml': {
26
- const { default: TTSGgml } = await import('@qvac/tts-ggml');
27
- return TTSGgml.assessFit(probe.request);
28
- }
29
- case 'audiogen-ggml': {
30
- const { assessFit } = await import('@qvac/audiogen-ggml');
31
- return assessFit(probe.request);
32
- }
33
- case 'diffusion-cpp': {
34
- const { assessFit } = await import('@qvac/diffusion-cpp');
35
- return assessFit(probe.request);
36
- }
37
- case 'ocr-ggml': {
38
- const { assessFit } = await import('@qvac/ocr-ggml');
39
- return assessFit(probe.request);
40
- }
41
- case 'translation-nmtcpp': {
42
- const { default: TranslationNmtcpp } = await import('@qvac/translation-nmtcpp');
43
- return TranslationNmtcpp.assessFit(probe.request);
44
- }
45
- }
46
- }
47
- export async function callChildEngineFit(probe) {
48
- return { engine: probe.engine, result: await assessFitOf(probe) };
49
- }
@@ -1,6 +0,0 @@
1
- import type { NativeProbeFit } from '../../../schemas/index.js';
2
- import type { FitProbeResult } from './engine-fit.js';
3
- export declare function classifyFit(probe: FitProbeResult, provenance: {
4
- basis: 'native-probe';
5
- estimatorVersion: string;
6
- }): NativeProbeFit;