@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -56,7 +56,8 @@ async function resolveBatchPrompts(prompts) {
56
56
  generationParams: prompt.generationParams
57
57
  }),
58
58
  ...(prompt.responseFormat && { responseFormat: prompt.responseFormat }),
59
- ...(allTools.length > 0 && { tools: allTools })
59
+ ...(allTools.length > 0 && { tools: allTools }),
60
+ ...(prompt.toolSearch && { toolSearch: prompt.toolSearch })
60
61
  });
61
62
  }
62
63
  return { prompts: resolvedPrompts, handlers: resolvedHandlers };
@@ -0,0 +1,30 @@
1
+ import { type CalcTokensParams, type RPCOptions, type Tool } from '../schemas/index.js';
2
+ import { type ToolInput } from '../utils/tool-helpers.js';
3
+ type OmitTools<T> = T extends unknown ? Omit<T, 'tools'> : never;
4
+ export type CalcTokensClientParams = OmitTools<CalcTokensParams> & {
5
+ tools?: Tool[] | ToolInput[];
6
+ };
7
+ /**
8
+ * Counts tokens with a loaded language model without running generation.
9
+ *
10
+ * When neither `history` nor enabled tools are supplied, `text` is counted raw.
11
+ * Otherwise, the conversation is rendered with the model's chat template and
12
+ * `text`, when present, becomes its final user turn. Tools are counted only when
13
+ * the model was loaded with `modelConfig.tools: true`.
14
+ * `toolSearch` controls the limits rendered for deferred-tool search, matching
15
+ * `completion()`.
16
+ * Media content and attachments are not supported.
17
+ * The count uses every supplied tool and the load-time reasoning setting. A named
18
+ * `generationParams.tool_choice` or `generationParams.reasoning_budget` override on
19
+ * `completion()` can render a different prompt.
20
+ *
21
+ * @param params - Loaded model ID and at least one prompt source: `text` or non-empty `history`.
22
+ * @param options - Dispatch options.
23
+ * @returns The number of tokens in the prompt under the model's load-time settings.
24
+ * @throws {RequestValidationFailedError} If the request is invalid.
25
+ * @throws {ModelNotFoundError} If `modelId` is not loaded.
26
+ * @throws {ModelOperationNotSupportedError} If the loaded model is not a language model.
27
+ * @throws {InvalidResponseError} If the handler returns a malformed result.
28
+ */
29
+ export declare function calcTokens(params: CalcTokensClientParams, options?: RPCOptions): Promise<number>;
30
+ export {};
@@ -0,0 +1,41 @@
1
+ import { send } from '../dispatch.js';
2
+ import { calcTokensParamsSchema, calcTokensResponseSchema } from '../schemas/index.js';
3
+ import { InvalidResponseError } from '../errors/index.js';
4
+ import { parseClientInput } from '../api/parse-input.js';
5
+ import { validateTools } from '../utils/tool-helpers.js';
6
+ /**
7
+ * Counts tokens with a loaded language model without running generation.
8
+ *
9
+ * When neither `history` nor enabled tools are supplied, `text` is counted raw.
10
+ * Otherwise, the conversation is rendered with the model's chat template and
11
+ * `text`, when present, becomes its final user turn. Tools are counted only when
12
+ * the model was loaded with `modelConfig.tools: true`.
13
+ * `toolSearch` controls the limits rendered for deferred-tool search, matching
14
+ * `completion()`.
15
+ * Media content and attachments are not supported.
16
+ * The count uses every supplied tool and the load-time reasoning setting. A named
17
+ * `generationParams.tool_choice` or `generationParams.reasoning_budget` override on
18
+ * `completion()` can render a different prompt.
19
+ *
20
+ * @param params - Loaded model ID and at least one prompt source: `text` or non-empty `history`.
21
+ * @param options - Dispatch options.
22
+ * @returns The number of tokens in the prompt under the model's load-time settings.
23
+ * @throws {RequestValidationFailedError} If the request is invalid.
24
+ * @throws {ModelNotFoundError} If `modelId` is not loaded.
25
+ * @throws {ModelOperationNotSupportedError} If the loaded model is not a language model.
26
+ * @throws {InvalidResponseError} If the handler returns a malformed result.
27
+ */
28
+ export async function calcTokens(params, options) {
29
+ const tools = params.tools ? validateTools(params.tools).tools : undefined;
30
+ const input = parseClientInput(calcTokensParamsSchema, {
31
+ ...params,
32
+ ...(tools !== undefined && { tools })
33
+ });
34
+ const response = await send({ ...input, type: 'calcTokens' }, options);
35
+ if (response.type !== 'calcTokens')
36
+ throw new InvalidResponseError('calcTokens');
37
+ const parsed = calcTokensResponseSchema.safeParse(response);
38
+ if (!parsed.success)
39
+ throw new InvalidResponseError('calcTokens', parsed.error);
40
+ return parsed.data.tokens;
41
+ }
@@ -2,7 +2,7 @@ import { type ClassifyClientParams, type ClassificationResult } from '../schemas
2
2
  /**
3
3
  * Classifies an image using a loaded classification model.
4
4
  *
5
- * The bundled MobileNetV3-Small model produces 3 labels: `"food"`, `"report"`, `"other"`.
5
+ * The `MOBILENETV3_3CLASS_V3_FP16` model produces 3 labels: `"food"`, `"report"`, `"other"`.
6
6
  * Custom models may emit different labels sourced from the GGUF metadata.
7
7
  *
8
8
  * @param params.modelId - The identifier of the loaded classification model
@@ -12,7 +12,10 @@ import { type ClassifyClientParams, type ClassificationResult } from '../schemas
12
12
  *
13
13
  * @example
14
14
  * ```typescript
15
- * const modelId = await loadModel({ modelType: "ggml-classification" });
15
+ * const modelId = await loadModel({
16
+ * modelSrc: MOBILENETV3_3CLASS_V3_FP16,
17
+ * modelType: "ggml-classification",
18
+ * });
16
19
  * const jpeg = fs.readFileSync("photo.jpg");
17
20
  * const results = await classify({ modelId, image: jpeg });
18
21
  * // [ { label: "food", confidence: 0.93 }, { label: "other", confidence: 0.05 }, ... ]
@@ -4,7 +4,7 @@ import { encodeBase64 } from '../utils/encoding.js';
4
4
  /**
5
5
  * Classifies an image using a loaded classification model.
6
6
  *
7
- * The bundled MobileNetV3-Small model produces 3 labels: `"food"`, `"report"`, `"other"`.
7
+ * The `MOBILENETV3_3CLASS_V3_FP16` model produces 3 labels: `"food"`, `"report"`, `"other"`.
8
8
  * Custom models may emit different labels sourced from the GGUF metadata.
9
9
  *
10
10
  * @param params.modelId - The identifier of the loaded classification model
@@ -14,7 +14,10 @@ import { encodeBase64 } from '../utils/encoding.js';
14
14
  *
15
15
  * @example
16
16
  * ```typescript
17
- * const modelId = await loadModel({ modelType: "ggml-classification" });
17
+ * const modelId = await loadModel({
18
+ * modelSrc: MOBILENETV3_3CLASS_V3_FP16,
19
+ * modelType: "ggml-classification",
20
+ * });
18
21
  * const jpeg = fs.readFileSync("photo.jpg");
19
22
  * const results = await classify({ modelId, image: jpeg });
20
23
  * // [ { label: "food", confidence: 0.93 }, { label: "other", confidence: 0.05 }, ... ]
@@ -45,7 +45,7 @@ type CompletionParams = Omit<CompletionClientParams, 'tools'> & {
45
45
  * validation must encode it explicitly in `schema`.
46
46
  *
47
47
  * Cannot be combined with `tools` (tools already constrain output via their parameter schema).
48
- * @param params.prefill - Evaluate the prompt and tools without generating an answer. Requires an enabled kvCache; cannot force tool_choice. Await final before sending the question with the same prefix.
48
+ * @param params.prefill - Evaluate the prompt and tools without generating an answer. Requires an enabled kvCache and at least one message (from history or `modelConfig.system_prompt`); cannot force tool_choice. Await final before sending the question with the same prefix.
49
49
  * @param params.kvCache - Optional KV cache configuration. Cache files are organized hierarchically:
50
50
  * - Structure: `{kvCacheKey}/{modelId}/{configHash}.bin`
51
51
  * - The configHash includes model config + system prompt to ensure cache isolation
@@ -45,7 +45,7 @@ const logger = getAppLogger();
45
45
  * validation must encode it explicitly in `schema`.
46
46
  *
47
47
  * Cannot be combined with `tools` (tools already constrain output via their parameter schema).
48
- * @param params.prefill - Evaluate the prompt and tools without generating an answer. Requires an enabled kvCache; cannot force tool_choice. Await final before sending the question with the same prefix.
48
+ * @param params.prefill - Evaluate the prompt and tools without generating an answer. Requires an enabled kvCache and at least one message (from history or `modelConfig.system_prompt`); cannot force tool_choice. Await final before sending the question with the same prefix.
49
49
  * @param params.kvCache - Optional KV cache configuration. Cache files are organized hierarchically:
50
50
  * - Structure: `{kvCacheKey}/{modelId}/{configHash}.bin`
51
51
  * - The configHash includes model config + system prompt to ensure cache isolation
@@ -186,6 +186,7 @@ export function completion(params) {
186
186
  kvCache: params.kvCache,
187
187
  prefill: params.prefill,
188
188
  tools: allTools.length > 0 ? allTools : undefined,
189
+ toolSearch: params.toolSearch,
189
190
  stream: params.stream ?? true,
190
191
  generationParams: params.generationParams,
191
192
  captureThinking: params.captureThinking,
@@ -1,11 +1,12 @@
1
1
  /**
2
- * Deletes KV cache files.
2
+ * Deletes KV cache files, or a registry model's cached download.
3
3
  *
4
4
  * @param params - The delete cache parameters
5
5
  * @param params.all - If true, deletes all cache files, including caller-owned named caches
6
6
  * @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
7
7
  * @param params.kvCacheKey - The cache key to delete
8
8
  * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
9
+ * @param params.modelSrc - A `registry://` model source. Deletes its cached files and the registry blocks a cancelled or failed download kept for resume. Fails while the model is downloading or loaded
9
10
  * @returns Promise resolving to success status
10
11
  * @throws {QvacErrorBase} When the cache parameters are invalid (`InvalidDeleteCacheParamsError`) or the engine reports a delete failure (`DeleteCacheFailedError`).
11
12
  * @example
@@ -21,6 +22,9 @@
21
22
  *
22
23
  * // Delete only specific model within cache key
23
24
  * await deleteCache({ kvCacheKey: "my-session", modelId: "model-abc123" });
25
+ *
26
+ * // Discard a paused registry download, or a cached registry model
27
+ * await deleteCache({ modelSrc: QWEN3_600M_INST_Q4.src });
24
28
  * ```
25
29
  */
26
30
  export declare function deleteCache(params: {
@@ -30,6 +34,8 @@ export declare function deleteCache(params: {
30
34
  } | {
31
35
  kvCacheKey: string;
32
36
  modelId?: string;
37
+ } | {
38
+ modelSrc: string;
33
39
  }): Promise<{
34
40
  success: boolean;
35
41
  }>;
@@ -1,13 +1,14 @@
1
1
  import { send } from '../dispatch.js';
2
2
  import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors/index.js';
3
3
  /**
4
- * Deletes KV cache files.
4
+ * Deletes KV cache files, or a registry model's cached download.
5
5
  *
6
6
  * @param params - The delete cache parameters
7
7
  * @param params.all - If true, deletes all cache files, including caller-owned named caches
8
8
  * @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
9
9
  * @param params.kvCacheKey - The cache key to delete
10
10
  * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
11
+ * @param params.modelSrc - A `registry://` model source. Deletes its cached files and the registry blocks a cancelled or failed download kept for resume. Fails while the model is downloading or loaded
11
12
  * @returns Promise resolving to success status
12
13
  * @throws {QvacErrorBase} When the cache parameters are invalid (`InvalidDeleteCacheParamsError`) or the engine reports a delete failure (`DeleteCacheFailedError`).
13
14
  * @example
@@ -23,6 +24,9 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
23
24
  *
24
25
  * // Delete only specific model within cache key
25
26
  * await deleteCache({ kvCacheKey: "my-session", modelId: "model-abc123" });
27
+ *
28
+ * // Discard a paused registry download, or a cached registry model
29
+ * await deleteCache({ modelSrc: QWEN3_600M_INST_Q4.src });
26
30
  * ```
27
31
  */
28
32
  export async function deleteCache(params) {
@@ -39,6 +43,12 @@ export async function deleteCache(params) {
39
43
  auto: true
40
44
  };
41
45
  }
46
+ else if ('modelSrc' in params) {
47
+ req = {
48
+ type: 'deleteCache',
49
+ modelSrc: params.modelSrc
50
+ };
51
+ }
42
52
  else if ('kvCacheKey' in params) {
43
53
  req = {
44
54
  type: 'deleteCache',
@@ -32,7 +32,7 @@ interface DiffusionResult {
32
32
  * const { outputs } = diffusion({ modelId, prompt: "oil painting style", init_image: initImage, strength: 0.7 });
33
33
  *
34
34
  * // img2img (FLUX.2 — in-context conditioning)
35
- * // IMPORTANT: FLUX img2img requires `prediction: "flux2_flow"` to be set on the
35
+ * // IMPORTANT: FLUX.2 img2img requires `prediction: "flux2_flow"` to be set on the
36
36
  * // model config at loadModel time (e.g. `loadModel(src, { modelType: "diffusion",
37
37
  * // modelConfig: { prediction: "flux2_flow" } })`).
38
38
  * const { outputs } = diffusion({ modelId, prompt: "turn into watercolor", init_image: initImage });
@@ -52,6 +52,19 @@ interface DiffusionResult {
52
52
  * height: 768,
53
53
  * });
54
54
  *
55
+ * // FLUX.3 Image Klein reference editing. Load the Qwen3-VL 4B encoder and the
56
+ * // FLUX.2 VAE; the engine detects FLUX.3 from the weights, so leave `prediction` unset.
57
+ * const { outputs: flux3Edit } = diffusion({
58
+ * modelId,
59
+ * prompt: "Place the subject in a lush garden, preserving its appearance.",
60
+ * init_images: [refA],
61
+ * width: 512,
62
+ * height: 512,
63
+ * steps: 16,
64
+ * cfg_scale: 1,
65
+ * sampling_method: "euler",
66
+ * });
67
+ *
55
68
  * // Qwen-Image-2.1 reference editing. Load the matching Qwen3-VL encoder,
56
69
  * // vision projector, and 2.1 VAE from your local files first.
57
70
  * const { outputs: edited } = diffusion({
@@ -24,7 +24,7 @@ import { decodeBase64, encodeBase64 } from '../utils/encoding.js';
24
24
  * const { outputs } = diffusion({ modelId, prompt: "oil painting style", init_image: initImage, strength: 0.7 });
25
25
  *
26
26
  * // img2img (FLUX.2 — in-context conditioning)
27
- * // IMPORTANT: FLUX img2img requires `prediction: "flux2_flow"` to be set on the
27
+ * // IMPORTANT: FLUX.2 img2img requires `prediction: "flux2_flow"` to be set on the
28
28
  * // model config at loadModel time (e.g. `loadModel(src, { modelType: "diffusion",
29
29
  * // modelConfig: { prediction: "flux2_flow" } })`).
30
30
  * const { outputs } = diffusion({ modelId, prompt: "turn into watercolor", init_image: initImage });
@@ -44,6 +44,19 @@ import { decodeBase64, encodeBase64 } from '../utils/encoding.js';
44
44
  * height: 768,
45
45
  * });
46
46
  *
47
+ * // FLUX.3 Image Klein reference editing. Load the Qwen3-VL 4B encoder and the
48
+ * // FLUX.2 VAE; the engine detects FLUX.3 from the weights, so leave `prediction` unset.
49
+ * const { outputs: flux3Edit } = diffusion({
50
+ * modelId,
51
+ * prompt: "Place the subject in a lush garden, preserving its appearance.",
52
+ * init_images: [refA],
53
+ * width: 512,
54
+ * height: 512,
55
+ * steps: 16,
56
+ * cfg_scale: 1,
57
+ * sampling_method: "euler",
58
+ * });
59
+ *
47
60
  * // Qwen-Image-2.1 reference editing. Load the matching Qwen3-VL encoder,
48
61
  * // vision projector, and 2.1 VAE from your local files first.
49
62
  * const { outputs: edited } = diffusion({
@@ -1,41 +1,94 @@
1
- import { type EmbedStats, type RPCOptions } from '../schemas/index.js';
1
+ import { send } from '../dispatch.js';
2
+ import { type EmbedClientParams, type EmbedBatchClientParams, type EmbedClientInput, type EmbedInput, type EmbedStats, type RPCOptions } from '../schemas/index.js';
3
+ type EmbedResult<T> = Promise<{
4
+ embedding: T;
5
+ stats?: EmbedStats;
6
+ }> & {
7
+ requestId: string;
8
+ };
9
+ /**
10
+ * Embeds one input supported by the loaded model.
11
+ *
12
+ * @overloadLabel "Single input"
13
+ * @param params - Model identifier and text, ordered text/media parts, or image bytes.
14
+ * @param options - Optional RPC options.
15
+ * @returns One vector, optional statistics, and a synchronous requestId for cancellation.
16
+ * @throws {QvacErrorBase} When validation or inference fails.
17
+ */
18
+ export declare function embed(params: {
19
+ modelId: string;
20
+ input: EmbedClientInput;
21
+ text?: never;
22
+ inputs?: never;
23
+ }, options?: RPCOptions): EmbedResult<number[]>;
2
24
  /**
3
- * Generates embeddings for a single text using a specified model.
25
+ * Embeds one text string.
4
26
  *
5
27
  * @overloadLabel "Single text"
6
- * @param params - The parameters for the embedding
7
- * @param params.modelId - The identifier of the embedding model to use
8
- * @param params.text - The input text to embed
9
- * @param options - Optional RPC options including per-call profiling
10
- * @returns A promise (decorated with `requestId`) resolving to an object with `embedding` (a single `number[]` vector) and optional `stats` performance data.
11
- * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
28
+ * @param params - Model identifier and text.
29
+ * @param options - Optional RPC options.
30
+ * @returns One vector with optional statistics and a requestId for cancellation.
31
+ * @throws {QvacErrorBase} When validation or inference fails.
12
32
  */
13
33
  export declare function embed(params: {
14
34
  modelId: string;
15
35
  text: string;
16
- }, options?: RPCOptions): Promise<{
17
- embedding: number[];
18
- stats?: EmbedStats;
19
- }> & {
20
- requestId: string;
21
- };
36
+ inputs?: never;
37
+ input?: never;
38
+ }, options?: RPCOptions): EmbedResult<number[]>;
22
39
  /**
23
- * Generates embeddings for multiple texts using a specified model.
24
- *
25
- * @overloadLabel "Multiple texts"
26
- * @param params - The parameters for the embedding
27
- * @param params.modelId - The identifier of the embedding model to use
28
- * @param params.text - The input texts to embed
29
- * @param options - Optional RPC options including per-call profiling
30
- * @returns A promise (decorated with `requestId`) resolving to an object with `embedding` (one `number[]` vector per input text, i.e. `number[][]`) and optional `stats` performance data.
31
- * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
40
+ * @deprecated Use `embedBatch({ modelId, inputs })` instead. This overload will be
41
+ * removed in the next major version.
42
+ *
43
+ * Embeds each string in a text array.
44
+ *
45
+ * @overloadLabel "Legacy text batch (deprecated)"
46
+ * @param params - Model identifier and text strings.
47
+ * @param options - Optional RPC options.
48
+ * @returns One vector per string, optional statistics, and a requestId.
49
+ * @throws {QvacErrorBase} When validation or inference fails.
32
50
  */
33
51
  export declare function embed(params: {
34
52
  modelId: string;
35
53
  text: string[];
36
- }, options?: RPCOptions): Promise<{
37
- embedding: number[][];
38
- stats?: EmbedStats;
39
- }> & {
40
- requestId: string;
41
- };
54
+ inputs?: never;
55
+ input?: never;
56
+ }, options?: RPCOptions): EmbedResult<number[][]>;
57
+ /**
58
+ * @deprecated Use `embedBatch({ modelId, inputs })` instead. This overload will be
59
+ * removed in the next major version.
60
+ *
61
+ * Embeds each text or ordered text/media input.
62
+ *
63
+ * @overloadLabel "Legacy inputs batch (deprecated)"
64
+ * @param params - Model identifier and text or ordered text/media inputs. Images go through embedBatch.
65
+ * @param options - Optional RPC options.
66
+ * @returns One vector per input, optional statistics, and a requestId.
67
+ * @throws {QvacErrorBase} When validation or inference fails.
68
+ */
69
+ export declare function embed(params: {
70
+ modelId: string;
71
+ inputs: EmbedInput[];
72
+ text?: never;
73
+ input?: never;
74
+ }, options?: RPCOptions): EmbedResult<number[][]>;
75
+ /**
76
+ * Embeds a single input or a legacy batch supported by the loaded model.
77
+ *
78
+ * @param params - Model identifier and input.
79
+ * @param options - Optional RPC options.
80
+ * @returns Embeddings, optional statistics, and a requestId for cancellation.
81
+ * @throws {QvacErrorBase} When validation or inference fails.
82
+ */
83
+ export declare function embed(params: EmbedClientParams, options?: RPCOptions): EmbedResult<number[] | number[][]>;
84
+ /**
85
+ * Embeds a batch of inputs supported by the loaded model, in input order.
86
+ *
87
+ * @param params - Model identifier and a non-empty list of inputs.
88
+ * @param options - Optional RPC options.
89
+ * @returns One vector per input, including one-item batches, optional statistics, and a requestId.
90
+ * @throws {QvacErrorBase} When validation or inference fails.
91
+ */
92
+ export declare function embedBatch(params: EmbedBatchClientParams, options?: RPCOptions): EmbedResult<number[][]>;
93
+ export declare function createEmbedCall(params: EmbedClientParams, options?: RPCOptions, sendRequest?: typeof send): EmbedResult<number[] | number[][]>;
94
+ export {};
package/dist/api/embed.js CHANGED
@@ -1,29 +1,75 @@
1
1
  import { send } from '../dispatch.js';
2
+ import { embedClientParamsSchema, embedBatchClientParamsSchema, embedResponseSchema, embedBatchResponseSchema } from '../schemas/index.js';
2
3
  import { InvalidResponseError } from '../errors/index.js';
4
+ import { parseClientInput } from '../api/parse-input.js';
5
+ import { encodeBase64 } from '../utils/encoding.js';
3
6
  import { decoratePromise } from '../utils/decorate-promise.js';
4
7
  import { generateRequestId } from '../runtime/request-id.js';
8
+ import { getAppLogger } from '../logging/index.js';
9
+ const logger = getAppLogger();
5
10
  export function embed(params, options) {
6
- // The caller-generated `requestId` is surfaced synchronously on the
7
- // returned promise so the caller can `cancel({ requestId })` before
8
- // `await` resolves. The same id is carried on the request so the
9
- // registry entry uses it as the canonical key — matching the
10
- // `loadModel` / `downloadAsset` / `completion` shape.
11
+ return createEmbedCall(params, options);
12
+ }
13
+ /**
14
+ * Embeds a batch of inputs supported by the loaded model, in input order.
15
+ *
16
+ * @param params - Model identifier and a non-empty list of inputs.
17
+ * @param options - Optional RPC options.
18
+ * @returns One vector per input, including one-item batches, optional statistics, and a requestId.
19
+ * @throws {QvacErrorBase} When validation or inference fails.
20
+ */
21
+ export function embedBatch(params, options) {
11
22
  const requestId = generateRequestId();
12
- const inner = runEmbed(params, requestId, options);
13
- return decoratePromise(inner, { requestId });
23
+ return decoratePromise(runEmbedBatch(params, requestId, options), { requestId });
14
24
  }
15
- async function runEmbed(params, requestId, options) {
16
- const request = {
17
- type: 'embed',
18
- ...params,
19
- requestId
20
- };
21
- const response = await send(request, options);
22
- if (response.type !== 'embed') {
23
- throw new InvalidResponseError('embed');
25
+ export function createEmbedCall(params, options, sendRequest = send) {
26
+ const requestId = generateRequestId();
27
+ return decoratePromise(runEmbed(params, requestId, options, sendRequest), { requestId });
28
+ }
29
+ function encodeInput(input) {
30
+ return typeof input === 'object' && !Array.isArray(input)
31
+ ? { ...input, image: encodeBase64(input.image) }
32
+ : input;
33
+ }
34
+ async function runEmbed(params, requestId, options, sendRequest) {
35
+ const parsed = parseClientInput(embedClientParamsSchema, params);
36
+ if (parsed.inputs !== undefined) {
37
+ logger.warn('embed() with inputs is deprecated. Use embedBatch() instead.');
24
38
  }
39
+ else if (Array.isArray(parsed.text)) {
40
+ logger.warn('embed() with a text array is deprecated. Use embedBatch() instead.');
41
+ }
42
+ const request = parsed.input !== undefined
43
+ ? { modelId: parsed.modelId, input: encodeInput(parsed.input) }
44
+ : parsed.inputs !== undefined
45
+ ? { modelId: parsed.modelId, inputs: parsed.inputs }
46
+ : { modelId: parsed.modelId, text: parsed.text };
47
+ const response = await sendRequest({ ...request, type: 'embed', requestId }, options);
48
+ if (response.type !== 'embed')
49
+ throw new InvalidResponseError('embed');
50
+ const result = embedResponseSchema.safeParse(response);
51
+ if (!result.success)
52
+ throw new InvalidResponseError('embed', result.error);
53
+ return {
54
+ embedding: result.data.embedding,
55
+ ...(result.data.stats !== undefined && { stats: result.data.stats })
56
+ };
57
+ }
58
+ async function runEmbedBatch(params, requestId, options) {
59
+ const parsed = parseClientInput(embedBatchClientParamsSchema, params);
60
+ const response = await send({
61
+ modelId: parsed.modelId,
62
+ inputs: parsed.inputs.map(encodeInput),
63
+ type: 'embedBatch',
64
+ requestId
65
+ }, options);
66
+ if (response.type !== 'embedBatch')
67
+ throw new InvalidResponseError('embedBatch');
68
+ const result = embedBatchResponseSchema.safeParse(response);
69
+ if (!result.success)
70
+ throw new InvalidResponseError('embedBatch', result.error);
25
71
  return {
26
- embedding: response.embedding,
27
- ...(response.stats !== undefined && { stats: response.stats })
72
+ embedding: result.data.embedding,
73
+ ...(result.data.stats !== undefined && { stats: result.data.stats })
28
74
  };
29
75
  }
@@ -12,7 +12,7 @@ export declare function getModelInfo(params: GetModelInfoParams): Promise<{
12
12
  modelId: string;
13
13
  expectedSize: number;
14
14
  sha256Checksum: string;
15
- addon: "embeddings" | "diffusion" | "audiogen" | "tts" | "ocr" | "vla" | "moss-transcribe" | "llm" | "whisper" | "bci" | "nmt" | "parakeet" | "classification" | "vad" | "other";
15
+ addon: "embeddings" | "diffusion" | "audiogen" | "tts" | "ocr" | "classification" | "vla" | "moss-transcribe" | "llm" | "whisper" | "bci" | "nmt" | "parakeet" | "vad" | "other";
16
16
  isCached: boolean;
17
17
  isLoaded: boolean;
18
18
  cacheFiles: {
@@ -3,13 +3,14 @@ export { downloadAsset } from '../api/download-asset.js';
3
3
  export { completion } from '../api/completion-stream.js';
4
4
  export { batchCompletion } from '../api/batch-completion.js';
5
5
  export { deleteCache } from '../api/delete-cache.js';
6
+ export { calcTokens } from '../api/calc-tokens.js';
6
7
  export { unloadModel } from '../api/unload-model.js';
7
8
  export { loggingStream } from '../api/logging-stream.js';
8
9
  export { subscribeServerLogs, type ServerLogHandler } from '../api/subscribe-logs.js';
9
10
  export { heartbeat } from '../api/heartbeat.js';
10
11
  export { transcribe, transcribeStream } from '../api/transcribe.js';
11
12
  export { bciTranscribe, bciTranscribeStream } from '../api/bci-transcribe.js';
12
- export { embed } from '../api/embed.js';
13
+ export { embed, embedBatch } from '../api/embed.js';
13
14
  export { finetune, type FinetuneHandle } from '../api/finetune.js';
14
15
  export { translate } from '../api/translate.js';
15
16
  export { cancel } from '../api/cancel.js';
package/dist/api/index.js CHANGED
@@ -3,13 +3,14 @@ export { downloadAsset } from '../api/download-asset.js';
3
3
  export { completion } from '../api/completion-stream.js';
4
4
  export { batchCompletion } from '../api/batch-completion.js';
5
5
  export { deleteCache } from '../api/delete-cache.js';
6
+ export { calcTokens } from '../api/calc-tokens.js';
6
7
  export { unloadModel } from '../api/unload-model.js';
7
8
  export { loggingStream } from '../api/logging-stream.js';
8
9
  export { subscribeServerLogs } from '../api/subscribe-logs.js';
9
10
  export { heartbeat } from '../api/heartbeat.js';
10
11
  export { transcribe, transcribeStream } from '../api/transcribe.js';
11
12
  export { bciTranscribe, bciTranscribeStream } from '../api/bci-transcribe.js';
12
- export { embed } from '../api/embed.js';
13
+ export { embed, embedBatch } from '../api/embed.js';
13
14
  export { finetune } from '../api/finetune.js';
14
15
  export { translate } from '../api/translate.js';
15
16
  export { cancel } from '../api/cancel.js';
package/dist/api/rag.d.ts CHANGED
@@ -89,7 +89,7 @@ export declare function ragIngest(params: RagIngestParams, options?: RPCOptions)
89
89
  * ```typescript
90
90
  * // Segregated flow
91
91
  * const chunks = await ragChunk({ documents: ["text1", "text2"] });
92
- * const { embedding: embeddings } = await embed({ modelId, text: chunks.map(c => c.content) });
92
+ * const { embedding: embeddings } = await embedBatch({ modelId, inputs: chunks.map(c => c.content) });
93
93
  * const embeddedDocs = chunks.map((chunk, i) => ({
94
94
  * ...chunk,
95
95
  * embedding: embeddings[i],
package/dist/api/rag.js CHANGED
@@ -156,7 +156,7 @@ async function runRagIngest(params, requestId, options) {
156
156
  * ```typescript
157
157
  * // Segregated flow
158
158
  * const chunks = await ragChunk({ documents: ["text1", "text2"] });
159
- * const { embedding: embeddings } = await embed({ modelId, text: chunks.map(c => c.content) });
159
+ * const { embedding: embeddings } = await embedBatch({ modelId, inputs: chunks.map(c => c.content) });
160
160
  * const embeddedDocs = chunks.map((chunk, i) => ({
161
161
  * ...chunk,
162
162
  * embedding: embeddings[i],
@@ -7,13 +7,19 @@ export interface ModelRegistrySearchParams {
7
7
  modelType?: ModelRegistryEntryAddon;
8
8
  addon?: ModelRegistryEntryAddon;
9
9
  }
10
+ /** A registry query's result, with the id that `cancel({ requestId })` takes. */
11
+ export type RegistryQuery<T> = Promise<T> & {
12
+ requestId: string;
13
+ };
10
14
  /**
11
15
  * Returns all available models from the QVAC distributed model registry.
12
16
  *
13
- * @returns A promise resolving to an array of `ModelRegistryEntry` describing every model the connected registry knows about.
17
+ * @returns A promise resolving to an array of `ModelRegistryEntry` describing every model the connected registry knows about,
18
+ * decorated with a synchronous `requestId` for `cancel({ requestId })` while the query waits for the registry.
19
+ * @throws {ModelRegistryBusyError} When another QVAC process kept the registry for longer than the query waits.
14
20
  * @throws {ModelRegistryQueryFailedError} When the registry query fails.
15
21
  */
16
- declare function modelRegistryList(): Promise<ModelRegistryEntry[]>;
22
+ declare function modelRegistryList(): RegistryQuery<ModelRegistryEntry[]>;
17
23
  /**
18
24
  * Searches the model registry with optional filters for model type, engine, and quantization.
19
25
  *
@@ -23,18 +29,22 @@ declare function modelRegistryList(): Promise<ModelRegistryEntry[]>;
23
29
  * @param params.quantization - Quantization identifier (e.g., `"Q4_K_M"`).
24
30
  * @param params.modelType - Alias for `addon`; kept for backward compatibility.
25
31
  * @param params.addon - Model addon / category to restrict results to.
26
- * @returns A promise resolving to the matching `ModelRegistryEntry` entries.
32
+ * @returns A promise resolving to the matching `ModelRegistryEntry` entries, decorated with a
33
+ * synchronous `requestId` for `cancel({ requestId })` while the query waits for the registry.
34
+ * @throws {ModelRegistryBusyError} When another QVAC process kept the registry for longer than the query waits.
27
35
  * @throws {ModelRegistryQueryFailedError} When the registry query fails.
28
36
  */
29
- declare function modelRegistrySearch(params?: ModelRegistrySearchParams): Promise<ModelRegistryEntry[]>;
37
+ declare function modelRegistrySearch(params?: ModelRegistrySearchParams): RegistryQuery<ModelRegistryEntry[]>;
30
38
  /**
31
39
  * Fetches a single model entry from the registry by its path and source.
32
40
  *
33
41
  * @param registryPath - Registry-relative path of the model to fetch.
34
42
  * @param registrySource - Registry source identifier (e.g., `"huggingface"`, `"local"`).
35
- * @returns A promise resolving to the matching `ModelRegistryEntry`.
43
+ * @returns A promise resolving to the matching `ModelRegistryEntry`, decorated with a
44
+ * synchronous `requestId` for `cancel({ requestId })` while the query waits for the registry.
36
45
  * @throws {ModelRegistryModelNotFoundError} When the registry has no model at this source and path.
46
+ * @throws {ModelRegistryBusyError} When another QVAC process kept the registry for longer than the query waits.
37
47
  * @throws {ModelRegistryQueryFailedError} When the registry query fails.
38
48
  */
39
- declare function modelRegistryGetModel(registryPath: string, registrySource: string): Promise<ModelRegistryEntry>;
49
+ declare function modelRegistryGetModel(registryPath: string, registrySource: string): RegistryQuery<ModelRegistryEntry>;
40
50
  export { modelRegistryList, modelRegistrySearch, modelRegistryGetModel };