@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../schemas/plugin.js';
53
54
  import { getModel, getModelConfig, getModelEntry } from '../../runtime/model-registry.js';
54
55
  import { ModelType } from '../../schemas/index.js';
55
56
  import { createAudioStream } from '../../utils/audio-input.js';
57
+ import { untilAborted } from '../../utils/until-aborted.js';
56
58
  import { getEngineLogger } from '../../logging/index.js';
57
- import { TranscriptionFailedError } from '../../errors/index.js';
59
+ import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
58
60
  import { nowMs } from '../../profiling/index.js';
59
61
  import { buildStreamResult } from '../../profiling/model-execution.js';
60
62
  import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
@@ -64,6 +66,9 @@ import { generateRandomRequestId } from '../../runtime/request-id.js';
64
66
  import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
65
67
  import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
66
68
  export { assertMetadataSupported, toTranscribeSegment };
69
+ async function* iterateAll(source) {
70
+ yield* source;
71
+ }
67
72
  function hasRunStreaming(model) {
68
73
  return 'runStreaming' in model && typeof model.runStreaming === 'function';
69
74
  }
@@ -173,10 +178,23 @@ function mapTranscribeStats(stats) {
173
178
  }),
174
179
  ...(stats?.encoderUsedCoreml !== undefined && {
175
180
  encoderUsedCoreml: stats.encoderUsedCoreml
181
+ }),
182
+ ...(stats?.aoscActive !== undefined && { aoscActive: stats.aoscActive }),
183
+ // MOSS-Transcribe-Diarize reports its counters under its own names. Its
184
+ // stage timings land on the shared encoder/decoder fields and its segment
185
+ // count on `totalSegments`; the token counts and prefill are MOSS-only.
186
+ ...(stats?.segments !== undefined && { totalSegments: stats.segments }),
187
+ ...(stats?.encodeMs !== undefined && { encoderTime: stats.encodeMs }),
188
+ ...(stats?.decodeMs !== undefined && { decoderTime: stats.decodeMs }),
189
+ ...(stats?.prefillMs !== undefined && { prefillTime: stats.prefillMs }),
190
+ ...(stats?.audioTokens !== undefined && { audioTokens: stats.audioTokens }),
191
+ ...(stats?.promptTokens !== undefined && { promptTokens: stats.promptTokens }),
192
+ ...(stats?.generatedTokens !== undefined && {
193
+ generatedTokens: stats.generatedTokens
176
194
  })
177
195
  };
178
196
  }
179
- export async function* transcribe(params, requestId) {
197
+ export async function* transcribe(params, requestId, sessionId, context) {
180
198
  const env_1 = { stack: [], error: void 0, hasError: false };
181
199
  try {
182
200
  const { modelId, metadata } = params;
@@ -186,9 +204,18 @@ export async function* transcribe(params, requestId) {
186
204
  // caller didn't send one.
187
205
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
188
206
  requestId: requestId ?? generateRandomRequestId(),
207
+ sessionId,
208
+ ...requestParent(context),
189
209
  kind: 'transcribe',
190
210
  modelId
191
- }), true);
211
+ })
212
+ // A request cancelled while it queued holds no slot; the model-wide
213
+ // `addon.cancel()` below would stop the job that does.
214
+ , true);
215
+ // A request cancelled while it queued holds no slot; the model-wide
216
+ // `addon.cancel()` below would stop the job that does.
217
+ if (ctx.signal.aborted)
218
+ throw new InferenceCancelledError(ctx.requestId);
192
219
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
193
220
  const engineType = getEngineModelType(modelId);
194
221
  const hasMossOptions = params.hotwords !== undefined || params.maxNewTokens !== undefined;
@@ -248,9 +275,7 @@ export async function* transcribe(params, requestId) {
248
275
  const chunks = (Array.isArray(output) ? output : [output]);
249
276
  if (metadata) {
250
277
  for (const chunk of chunks) {
251
- if (!chunk.text)
252
- continue;
253
- if (silenceMarker && chunk.text.includes(silenceMarker))
278
+ if (!carriesSegment(chunk, silenceMarker))
254
279
  continue;
255
280
  yield toTranscribeSegment(chunk);
256
281
  }
@@ -297,7 +322,7 @@ function buildRunStreamingOpts(engineType, opts) {
297
322
  }
298
323
  return runOpts;
299
324
  }
300
- export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
325
+ export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId, sessionId, context) {
301
326
  const env_2 = { stack: [], error: void 0, hasError: false };
302
327
  try {
303
328
  // Same `kind: "transcribe"` as the unary variant — the registry
@@ -306,9 +331,13 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
306
331
  // either shape.
307
332
  const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
308
333
  requestId: requestId ?? generateRandomRequestId(),
334
+ sessionId,
335
+ ...requestParent(context),
309
336
  kind: 'transcribe',
310
337
  modelId
311
338
  }), true);
339
+ if (ctx.signal.aborted)
340
+ throw new InferenceCancelledError(ctx.requestId);
312
341
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
313
342
  const engineType = getEngineModelType(modelId);
314
343
  assertMetadataSupported(modelId, engineType, metadata);
@@ -321,26 +350,19 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
321
350
  if (!hasRunStreaming(model)) {
322
351
  throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
323
352
  }
324
- const onAbort = () => {
325
- const addon = model.addon;
326
- if (addon?.cancel) {
327
- addon.cancel.call(addon).catch((err) => {
328
- requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
329
- });
330
- }
331
- };
332
- ctx.signal.addEventListener('abort', onAbort, { once: true });
333
- if (ctx.signal.aborted)
334
- onAbort();
335
- ctx.scope.defer(() => {
336
- ctx.signal.removeEventListener('abort', onAbort);
337
- });
353
+ // A streaming session ends when its input ends, so an abort ends the input
354
+ // and the session drains the audio it already has, yielding nothing more.
355
+ // The model-wide `addon.cancel()` the unary path uses is not safe here: a
356
+ // parakeet stream cancelled that way leaves its stream state to the next
357
+ // session, whose output starts with the cancelled session's audio and never
358
+ // ends.
359
+ const input = untilAborted(iterateAll(audioInputStream), ctx.signal);
338
360
  const runOpts = buildRunStreamingOpts(engineType, opts);
339
361
  const modelStart = nowMs();
340
- const response = await model.runStreaming(audioInputStream, runOpts);
362
+ const response = await model.runStreaming(input, runOpts);
341
363
  for await (const output of response.iterate()) {
342
364
  if (ctx.signal.aborted)
343
- break;
365
+ continue;
344
366
  requestLogger.debug('Live Transcription Update:', output);
345
367
  if (!Array.isArray(output)) {
346
368
  if (isVadEvent(output)) {
@@ -386,17 +408,30 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
386
408
  await result_2;
387
409
  }
388
410
  }
411
+ /**
412
+ * Whether a native segment becomes a metadata segment. Text-less segments are
413
+ * dropped, except an explicit final marker: parakeet streaming closes a chunk
414
+ * with `isFinal: true` and no text, and that marker is the only finality signal
415
+ * the caller gets for the chunk.
416
+ */
417
+ function carriesSegment(segment, silenceMarker) {
418
+ if (!segment.text)
419
+ return segment.isFinal === true;
420
+ return !silenceMarker || !segment.text.includes(silenceMarker);
421
+ }
389
422
  function* emitSegment(segment, metadata, silenceMarker) {
423
+ if (metadata) {
424
+ if (carriesSegment(segment, silenceMarker))
425
+ yield toTranscribeSegment(segment);
426
+ return;
427
+ }
390
428
  if (!segment.text) {
391
429
  return;
392
430
  }
393
431
  if (silenceMarker && segment.text.includes(silenceMarker)) {
394
432
  return;
395
433
  }
396
- if (metadata) {
397
- yield toTranscribeSegment(segment);
398
- }
399
- else if (segment.text.trim()) {
434
+ if (segment.text.trim()) {
400
435
  yield segment.text;
401
436
  }
402
437
  }
@@ -1,7 +1,8 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type TranslateParams, type TranslationStats } from '../../schemas/index.js';
2
3
  export declare function getLanguage(code: string | undefined): string;
3
4
  export declare function isAfrican(code: string | undefined): boolean;
4
- export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
5
+ export declare function translate(params: TranslateParams, requestId?: string, sessionId?: string, invoke?: PluginHandlerContext): AsyncGenerator<string, {
5
6
  modelExecutionMs: number;
6
7
  stats?: TranslationStats;
7
8
  }, unknown>;
@@ -50,6 +50,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../schemas/plugin.js';
53
54
  import { getModelEntry } from '../../runtime/model-registry.js';
54
55
  import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LANGUAGES_MAP } from '../../schemas/index.js';
55
56
  import { getLangName, detectOne } from '@qvac/langdetect-text';
@@ -122,7 +123,7 @@ const LLM_TRANSLATE_GENERATION_PARAMS = {
122
123
  function shouldSkipPerCallSampling(modelName) {
123
124
  return !!modelName && modelName.startsWith('AFRICAN_');
124
125
  }
125
- export async function* translate(params, requestId) {
126
+ export async function* translate(params, requestId, sessionId, invoke) {
126
127
  const env_1 = { stack: [], error: void 0, hasError: false };
127
128
  try {
128
129
  const { modelId, text, modelType: inputModelType } = params;
@@ -168,6 +169,8 @@ export async function* translate(params, requestId) {
168
169
  // fine because the result is dropped either way.
169
170
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
170
171
  requestId: requestId ?? generateRandomRequestId(),
172
+ sessionId,
173
+ ...requestParent(invoke),
171
174
  kind: 'translate',
172
175
  modelId,
173
176
  // LLM translate joins the completion lane (its cap = the model's own
@@ -1,2 +1,18 @@
1
1
  import { type UnloadModelParams } from '../../schemas/index.js';
2
- export declare function unloadModel(params: UnloadModelParams): Promise<void>;
2
+ export interface UnloadModelResult {
3
+ /** The caller's lease was dropped. False when the caller held none. */
4
+ released: boolean;
5
+ /** The model left memory. False while another session still holds a lease. */
6
+ unloaded: boolean;
7
+ }
8
+ /**
9
+ * Releases the caller session's lease on the model. The model is torn down
10
+ * only when no lease remains; a session that never loaded the model cannot
11
+ * unload it from under the sessions that did.
12
+ *
13
+ * Runs under the model's load lock: a load of the same configuration that
14
+ * arrives during the teardown waits for it and then loads a fresh copy,
15
+ * instead of finding the id unregistered and loading a second native model
16
+ * while the first is still being freed.
17
+ */
18
+ export declare function unloadModel(params: UnloadModelParams): Promise<UnloadModelResult>;
@@ -1,7 +1,8 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { unregisterModel } from '../../runtime/model-registry.js';
3
+ import { getModelEntry, releaseModelLease, unregisterModel } from '../../runtime/model-registry.js';
4
4
  import { getRequestRegistry } from '../../runtime/index.js';
5
+ import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
5
6
  import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
6
7
  import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
7
8
  import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
@@ -9,9 +10,36 @@ import { unloadModelParamsSchema } from '../../schemas/index.js';
9
10
  import { ModelNotLoadedError } from '../../errors/index.js';
10
11
  import { detectShardedModel } from '../../utils/index.js';
11
12
  import { getClearStorageTarget } from '../../utils/cache/paths.js';
13
+ import { withModelWorkLock } from '../../runtime/model-work-lock.js';
12
14
  const logger = getEngineLogger();
15
+ /**
16
+ * Releases the caller session's lease on the model. The model is torn down
17
+ * only when no lease remains; a session that never loaded the model cannot
18
+ * unload it from under the sessions that did.
19
+ *
20
+ * Runs under the model's load lock: a load of the same configuration that
21
+ * arrives during the teardown waits for it and then loads a fresh copy,
22
+ * instead of finding the id unregistered and loading a second native model
23
+ * while the first is still being freed.
24
+ */
13
25
  export async function unloadModel(params) {
14
- const { modelId, clearStorage } = unloadModelParamsSchema.parse(params);
26
+ const parsed = unloadModelParamsSchema.parse(params);
27
+ return withModelWorkLock(parsed.modelId, () => unloadModelLocked(parsed));
28
+ }
29
+ async function unloadModelLocked(params) {
30
+ const { modelId, clearStorage, sessionId = OWNER_SESSION_ID } = params;
31
+ if (!getModelEntry(modelId)) {
32
+ throw new ModelNotLoadedError(modelId);
33
+ }
34
+ const remaining = releaseModelLease(modelId, sessionId);
35
+ if (remaining === null) {
36
+ logger.info(`Session ${sessionId} holds no lease on model ${modelId}; nothing released`);
37
+ return { released: false, unloaded: false };
38
+ }
39
+ if (remaining > 0) {
40
+ logger.info(`Model ${modelId} lease released by ${sessionId}; ${remaining} lease(s) remain`);
41
+ return { released: true, unloaded: false };
42
+ }
15
43
  const entry = unregisterModel(modelId);
16
44
  if (!entry) {
17
45
  throw new ModelNotLoadedError(modelId);
@@ -46,4 +74,5 @@ export async function unloadModel(params) {
46
74
  unregisterAllLoggingStreams(modelId);
47
75
  logger.info(`Model ${modelId} unloaded`);
48
76
  });
77
+ return { released: true, unloaded: true };
49
78
  }
@@ -11,7 +11,7 @@ export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStora
11
11
  * The vector index feature's only backend: the fabric vector index that the
12
12
  * embedding plugin exposes through the `turbovecIndexProvider` capability.
13
13
  * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
14
- * dimension rule (multiple of 8, at most 1024), which the native index
14
+ * dimension rule (multiple of 8, at most 16384), which the native index
15
15
  * enforces itself.
16
16
  */
17
17
  export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
@@ -48,6 +48,21 @@ function openIndex(index, storage) {
48
48
  return index.length;
49
49
  },
50
50
  storage,
51
+ get calibrated() {
52
+ return index.calibrated;
53
+ },
54
+ get stagedSearch() {
55
+ return index.stagedSearch;
56
+ },
57
+ setStagedSearch(enabled) {
58
+ // Without the getter the setting could not be confirmed, so a provider
59
+ // needs both halves of the control.
60
+ if (!index.setStagedSearch || typeof index.stagedSearch !== 'boolean') {
61
+ throw new Error('The vector index provider does not support staged search controls');
62
+ }
63
+ index.setStagedSearch(enabled);
64
+ return index.stagedSearch;
65
+ },
51
66
  add(vectors, ids) {
52
67
  index.addWithIds(vectors, ids);
53
68
  needsPrepare = true;
@@ -80,7 +95,7 @@ function openIndex(index, storage) {
80
95
  * The vector index feature's only backend: the fabric vector index that the
81
96
  * embedding plugin exposes through the `turbovecIndexProvider` capability.
82
97
  * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
83
- * dimension rule (multiple of 8, at most 1024), which the native index
98
+ * dimension rule (multiple of 8, at most 16384), which the native index
84
99
  * enforces itself.
85
100
  */
86
101
  export function getTurboVecBackend() {
@@ -54,7 +54,8 @@ export function buildOperationEvent(op, profileId, ts, executionMs, request, fin
54
54
  Object.assign(gauges, extracted);
55
55
  }
56
56
  const requestTags = config.getTags?.(request);
57
- const resultTags = config.getTagsFromResult?.(finalResponse);
57
+ // A handler that threw has no response; its event carries request tags only.
58
+ const resultTags = finalResponse ? config.getTagsFromResult?.(finalResponse) : undefined;
58
59
  const tags = { ...requestTags, ...resultTags };
59
60
  const hasGauges = Object.keys(gauges).length > 0;
60
61
  const hasTags = Object.keys(tags).length > 0;
@@ -257,25 +258,27 @@ registerOperationMetrics({
257
258
  return Object.keys(gauges).length > 0 ? gauges : undefined;
258
259
  }
259
260
  });
260
- registerOperationMetrics({
261
- op: 'embed',
262
- kind: 'handler',
263
- getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
264
- fromResult: (res) => {
265
- const gauges = {};
266
- const modelExecMs = readModelExecutionMs(res);
267
- if (modelExecMs !== undefined)
268
- gauges['modelExecutionTime'] = modelExecMs;
269
- if (res.stats?.totalTime !== undefined)
270
- gauges['totalTime'] = res.stats.totalTime;
271
- if (res.stats?.tokensPerSecond !== undefined) {
272
- gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
261
+ for (const op of ['embed', 'embedBatch']) {
262
+ registerOperationMetrics({
263
+ op,
264
+ kind: 'handler',
265
+ getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
266
+ fromResult: (res) => {
267
+ const gauges = {};
268
+ const modelExecMs = readModelExecutionMs(res);
269
+ if (modelExecMs !== undefined)
270
+ gauges['modelExecutionTime'] = modelExecMs;
271
+ if (res.stats?.totalTime !== undefined)
272
+ gauges['totalTime'] = res.stats.totalTime;
273
+ if (res.stats?.tokensPerSecond !== undefined) {
274
+ gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
275
+ }
276
+ if (res.stats?.totalTokens !== undefined)
277
+ gauges['totalTokens'] = res.stats.totalTokens;
278
+ return Object.keys(gauges).length > 0 ? gauges : undefined;
273
279
  }
274
- if (res.stats?.totalTokens !== undefined)
275
- gauges['totalTokens'] = res.stats.totalTokens;
276
- return Object.keys(gauges).length > 0 ? gauges : undefined;
277
- }
278
- });
280
+ });
281
+ }
279
282
  registerOperationMetrics({
280
283
  op: 'ocrStream',
281
284
  kind: 'handler',
@@ -0,0 +1,16 @@
1
+ import { type EmbeddingFunction } from '@qvac/rag';
2
+ import type { PluginHandlerContext } from '../schemas/plugin.js';
3
+ export declare function setRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
4
+ /** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
5
+ export declare function clearRagEmbedParent(workspace: string | undefined, parent: PluginHandlerContext): void;
6
+ export declare function getRagEmbedParent(workspace: string | undefined): PluginHandlerContext | undefined;
7
+ /**
8
+ * The text the workspace embedding function is asked for when a search runs:
9
+ * `@qvac/rag` trims a query before it embeds it, so a search embeds and
10
+ * stashes the trimmed text to be found by the same key.
11
+ */
12
+ export declare function queryEmbeddingText(query: string): string;
13
+ export declare function stashQueryEmbedding(workspace: string | undefined, query: string, vector: number[]): void;
14
+ export declare function takeQueryEmbedding(workspace: string | undefined, query: string): number[] | undefined;
15
+ /** The embedding function a workspace's RAG instance is created with. */
16
+ export declare function createWorkspaceEmbeddingFunction(modelId: string, workspace: string | undefined): EmbeddingFunction;
@@ -0,0 +1,81 @@
1
+ import { ERR_CODES, QvacErrorRAG } from '@qvac/rag';
2
+ import { InferenceCancelledError } from '../errors/index.js';
3
+ import { embed } from '../plugins/ops/embed.js';
4
+ import { getWorkspaceKey } from '../rag/rag-operation-manager.js';
5
+ /**
6
+ * The request a workspace's embeddings run under.
7
+ *
8
+ * A workspace holds one RAG instance with one embedding function for its
9
+ * lifetime, so the function cannot be bound to the request that is using it.
10
+ * Instead, the registered RAG operation on the workspace (`ingest`,
11
+ * `saveEmbeddings`, `reindex`; one at a time, see `beginRagContext`) publishes
12
+ * itself here for its duration, and the embedding function begins every
13
+ * `embed` as a child of it: the embedding runs as the caller's session and
14
+ * aborts with the operation. A `search` is not a registered operation and
15
+ * runs beside them, so it embeds its query itself, under its own request, and
16
+ * leaves the vector here for the function to hand back.
17
+ */
18
+ const parentByWorkspace = new Map();
19
+ const queryVectorsByWorkspace = new Map();
20
+ export function setRagEmbedParent(workspace, parent) {
21
+ parentByWorkspace.set(getWorkspaceKey(workspace), parent);
22
+ }
23
+ /** Clears the parent only while it is still the one set, so an older operation's unwind never drops a newer one. */
24
+ export function clearRagEmbedParent(workspace, parent) {
25
+ const key = getWorkspaceKey(workspace);
26
+ if (parentByWorkspace.get(key) === parent)
27
+ parentByWorkspace.delete(key);
28
+ }
29
+ export function getRagEmbedParent(workspace) {
30
+ return parentByWorkspace.get(getWorkspaceKey(workspace));
31
+ }
32
+ /**
33
+ * The text the workspace embedding function is asked for when a search runs:
34
+ * `@qvac/rag` trims a query before it embeds it, so a search embeds and
35
+ * stashes the trimmed text to be found by the same key.
36
+ */
37
+ export function queryEmbeddingText(query) {
38
+ return query.trim();
39
+ }
40
+ export function stashQueryEmbedding(workspace, query, vector) {
41
+ const key = getWorkspaceKey(workspace);
42
+ let vectors = queryVectorsByWorkspace.get(key);
43
+ if (!vectors) {
44
+ vectors = new Map();
45
+ queryVectorsByWorkspace.set(key, vectors);
46
+ }
47
+ vectors.set(query, vector);
48
+ }
49
+ export function takeQueryEmbedding(workspace, query) {
50
+ const key = getWorkspaceKey(workspace);
51
+ const vectors = queryVectorsByWorkspace.get(key);
52
+ const vector = vectors?.get(query);
53
+ if (vectors && vector !== undefined) {
54
+ vectors.delete(query);
55
+ if (vectors.size === 0)
56
+ queryVectorsByWorkspace.delete(key);
57
+ }
58
+ return vector;
59
+ }
60
+ /** The embedding function a workspace's RAG instance is created with. */
61
+ export function createWorkspaceEmbeddingFunction(modelId, workspace) {
62
+ return async (text) => {
63
+ if (typeof text === 'string') {
64
+ const stashed = takeQueryEmbedding(workspace, text);
65
+ if (stashed !== undefined)
66
+ return stashed;
67
+ }
68
+ try {
69
+ const result = await embed({ modelId, text }, undefined, undefined, getRagEmbedParent(workspace));
70
+ return result.embedding;
71
+ }
72
+ catch (error) {
73
+ // `@qvac/rag` reports any other error from this function as a failed
74
+ // generation, so a cancelled embed is handed back as its cancellation.
75
+ if (error instanceof InferenceCancelledError) {
76
+ throw new QvacErrorRAG({ code: ERR_CODES.OPERATION_CANCELLED, cause: error });
77
+ }
78
+ throw error;
79
+ }
80
+ };
81
+ }
@@ -1,2 +1,27 @@
1
1
  import type { RagRequest, RagResponse, RagProgressUpdate } from '../schemas/index.js';
2
+ import { type ManagedRequestContext } from '../runtime/index.js';
3
+ /**
4
+ * Begin a registry-tracked RAG context with workspace-level pre-emption.
5
+ *
6
+ * Workspace-level admission lives in the dispatcher rather than as a
7
+ * registry policy primitive (it's a dispatch concern, not a registry
8
+ * `kind` admission rule). The sequence is **cancel-prior → begin-new**:
9
+ * if another RAG operation is already running on the same workspace,
10
+ * cancel it first, then begin the new context. Reversing the order
11
+ * would cancel the just-installed context.
12
+ *
13
+ * The workspace → requestId map is updated after `begin(...)` succeeds
14
+ * and cleared on scope unwind via `scope.defer(...)`, with a
15
+ * "still mine?" guard so an older op's deferred cleanup cannot stomp
16
+ * a newer op's mapping.
17
+ *
18
+ * A session pre-empts only its own operation: one that finds another
19
+ * session's operation on the workspace is refused with
20
+ * `RAGWorkspaceBusyError`, since a session reaches only its own requests.
21
+ * The owner pre-empts any session's operation.
22
+ */
23
+ export declare function beginRagContext(workspace: string, request: {
24
+ requestId?: string | undefined;
25
+ sessionId?: string | undefined;
26
+ }): Promise<ManagedRequestContext>;
2
27
  export declare function handleRag(request: RagRequest, onProgress?: (update: RagProgressUpdate) => void): Promise<RagResponse>;
@@ -53,6 +53,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
53
53
  import { chunk, ingest, reindex, saveEmbeddings, search, deleteEmbeddings, listWorkspaces, closeWorkspace, deleteWorkspace, DEFAULT_WORKSPACE, getActiveRagRequest, setActiveRagRequest, clearActiveRagRequest } from '../rag/index.js';
54
54
  import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
55
55
  import { generateRandomRequestId } from '../runtime/request-id.js';
56
+ import { OWNER_SESSION_ID, resolveSessionId } from '../runtime/session-id.js';
57
+ import { RAGWorkspaceBusyError } from '../errors/index.js';
58
+ import { clearRagEmbedParent, setRagEmbedParent } from '../rag/embedding-scope.js';
56
59
  import { getEngineLogger } from '../logging/index.js';
57
60
  import { profileReplyHandler, registerOperationMetrics } from '../profiling/index.js';
58
61
  registerOperationMetrics({
@@ -110,20 +113,51 @@ function omitOnProgress(obj) {
110
113
  * and cleared on scope unwind via `scope.defer(...)`, with a
111
114
  * "still mine?" guard so an older op's deferred cleanup cannot stomp
112
115
  * a newer op's mapping.
116
+ *
117
+ * A session pre-empts only its own operation: one that finds another
118
+ * session's operation on the workspace is refused with
119
+ * `RAGWorkspaceBusyError`, since a session reaches only its own requests.
120
+ * The owner pre-empts any session's operation.
113
121
  */
114
- async function beginRagContext(workspace, requestId) {
122
+ export async function beginRagContext(workspace, request) {
115
123
  const registry = getRequestRegistry();
124
+ const requestId = request.requestId ?? generateRandomRequestId();
125
+ const sessionId = resolveSessionId(request);
126
+ const identity = { requestId, sessionId };
116
127
  const prev = getActiveRagRequest(workspace);
117
- if (prev !== undefined && prev !== requestId) {
118
- registry.cancel({ requestId: prev, reason: 'rag-workspace-preempt' });
128
+ // The same request sent again: the registry refuses the duplicate id, and
129
+ // the running operation keeps the workspace.
130
+ const duplicate = prev?.requestId === requestId && prev.sessionId === sessionId;
131
+ if (prev !== undefined && !duplicate) {
132
+ if (prev.sessionId !== sessionId && sessionId !== OWNER_SESSION_ID) {
133
+ throw new RAGWorkspaceBusyError(workspace);
134
+ }
135
+ registry.cancel({ ...prev, reason: 'rag-workspace-preempt' });
119
136
  }
120
- const ctx = await registry.begin({
121
- requestId,
122
- kind: 'rag'
123
- });
124
- setActiveRagRequest(workspace, requestId);
137
+ // Recorded before the admission wait, so a second call arriving during it
138
+ // sees this operation and is refused or pre-empts it.
139
+ if (!duplicate)
140
+ setActiveRagRequest(workspace, identity);
141
+ let ctx;
142
+ try {
143
+ ctx = await registry.begin({
144
+ requestId,
145
+ sessionId,
146
+ kind: 'rag'
147
+ });
148
+ }
149
+ catch (error) {
150
+ if (!duplicate)
151
+ clearActiveRagRequest(workspace, identity);
152
+ throw error;
153
+ }
154
+ // The workspace's embeddings run as children of this request for its
155
+ // duration: the caller's session, aborted with it.
156
+ const embedParent = { signal: ctx.signal, requestId, sessionId: request.sessionId };
157
+ setRagEmbedParent(workspace, embedParent);
125
158
  ctx.scope.defer(() => {
126
- clearActiveRagRequest(workspace, requestId);
159
+ clearRagEmbedParent(workspace, embedParent);
160
+ clearActiveRagRequest(workspace, identity);
127
161
  });
128
162
  return ctx;
129
163
  }
@@ -145,8 +179,7 @@ async function handleRagInternal(request, onProgress) {
145
179
  const env_1 = { stack: [], error: void 0, hasError: false };
146
180
  try {
147
181
  const workspace = request.workspace ?? DEFAULT_WORKSPACE;
148
- const requestId = request.requestId ?? generateRandomRequestId();
149
- const ctx = __addDisposableResource(env_1, await beginRagContext(workspace, requestId), true);
182
+ const ctx = __addDisposableResource(env_1, await beginRagContext(workspace, request), true);
150
183
  const log = withRequestContext(getEngineLogger(), ctx);
151
184
  log.debug('ingest start');
152
185
  const handlerOptions = createHandlerOptions('ingest', workspace, ctx.signal, onProgress);
@@ -174,8 +207,7 @@ async function handleRagInternal(request, onProgress) {
174
207
  const env_2 = { stack: [], error: void 0, hasError: false };
175
208
  try {
176
209
  const workspace = request.workspace ?? DEFAULT_WORKSPACE;
177
- const requestId = request.requestId ?? generateRandomRequestId();
178
- const ctx = __addDisposableResource(env_2, await beginRagContext(workspace, requestId), true);
210
+ const ctx = __addDisposableResource(env_2, await beginRagContext(workspace, request), true);
179
211
  const log = withRequestContext(getEngineLogger(), ctx);
180
212
  log.debug('saveEmbeddings start');
181
213
  const handlerOptions = createHandlerOptions('saveEmbeddings', workspace, ctx.signal, onProgress);
@@ -199,7 +231,10 @@ async function handleRagInternal(request, onProgress) {
199
231
  }
200
232
  }
201
233
  case 'search': {
202
- const results = await search(request);
234
+ const results = await search(request, {
235
+ requestId: request.requestId,
236
+ sessionId: request.sessionId
237
+ });
203
238
  return {
204
239
  type: 'rag',
205
240
  operation: request.operation,
@@ -219,8 +254,7 @@ async function handleRagInternal(request, onProgress) {
219
254
  const env_3 = { stack: [], error: void 0, hasError: false };
220
255
  try {
221
256
  const workspace = request.workspace ?? DEFAULT_WORKSPACE;
222
- const requestId = request.requestId ?? generateRandomRequestId();
223
- const ctx = __addDisposableResource(env_3, await beginRagContext(workspace, requestId), true);
257
+ const ctx = __addDisposableResource(env_3, await beginRagContext(workspace, request), true);
224
258
  const log = withRequestContext(getEngineLogger(), ctx);
225
259
  log.debug('reindex start');
226
260
  const handlerOptions = createHandlerOptions('reindex', workspace, ctx.signal, onProgress);
@@ -1,13 +1,9 @@
1
1
  import { getRagInstance } from '../../rag/rag-workspace-manager.js';
2
- import { embed } from '../../plugins/ops/embed.js';
2
+ import { createWorkspaceEmbeddingFunction } from '../../rag/embedding-scope.js';
3
3
  import { ragIngestParamsSchema } from '../../schemas/index.js';
4
4
  export async function ingest(params, options) {
5
5
  const { modelId, documents, chunk, chunkOpts, workspace, progressInterval } = ragIngestParamsSchema.parse(params);
6
- async function embeddingFunction(text) {
7
- const result = await embed({ modelId, text });
8
- return result.embedding;
9
- }
10
- const rag = await getRagInstance(modelId, embeddingFunction, workspace);
6
+ const rag = await getRagInstance(modelId, createWorkspaceEmbeddingFunction(modelId, workspace), workspace);
11
7
  const ingestOpts = {
12
8
  chunk,
13
9
  chunkOpts,