@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -41,6 +41,8 @@ export interface DownloadRequestBinding {
41
41
  signal: AbortSignal;
42
42
  scope: DisposableScope;
43
43
  requestId: string;
44
+ /** Session the request runs as; the registry resolves `requestId` within it. */
45
+ sessionId: string;
44
46
  }
45
47
  export interface DownloadHooks {
46
48
  onDownloadKey?: (key: string) => void;
@@ -1,2 +1,3 @@
1
1
  import type { LoggingStreamRequest, LoggingStreamResponse } from '../schemas/index.js';
2
- export declare function handleLoggingStream(request: LoggingStreamRequest): AsyncGenerator<LoggingStreamResponse>;
2
+ import type { StreamHandlerContext } from '../handlers/types.js';
3
+ export declare function handleLoggingStream(request: LoggingStreamRequest, context?: StreamHandlerContext): AsyncGenerator<LoggingStreamResponse>;
@@ -1,8 +1,27 @@
1
1
  import { registerLoggingStream, unregisterLoggingStream } from '../runtime/logging-stream-registry.js';
2
- export async function* handleLoggingStream(request) {
2
+ import { isOwnerSession, isSessionOpen, onSessionClose } from '../runtime/session-registry.js';
3
+ import { resolveSessionId } from '../runtime/session-id.js';
4
+ export async function* handleLoggingStream(request, context = {}) {
3
5
  const { id } = request;
6
+ const { signal } = context;
7
+ if (signal?.aborted)
8
+ return;
9
+ // A subscription is not a registered request, so the session close that
10
+ // drains requests would not reach it; it ends on its own close listener.
11
+ const sessionId = resolveSessionId(request);
12
+ const scoped = !isOwnerSession(sessionId);
13
+ if (scoped && !isSessionOpen(sessionId))
14
+ return;
15
+ let stopped = false;
4
16
  const logQueue = [];
5
17
  let pendingResolve = null;
18
+ const wake = () => {
19
+ if (pendingResolve) {
20
+ const resolve = pendingResolve;
21
+ pendingResolve = null;
22
+ resolve();
23
+ }
24
+ };
6
25
  const streamHandler = (level, namespace, message, sourceId) => {
7
26
  const logResponse = {
8
27
  type: 'loggingStream',
@@ -15,25 +34,34 @@ export async function* handleLoggingStream(request) {
15
34
  timestamp: Date.now()
16
35
  };
17
36
  logQueue.push(logResponse);
18
- if (pendingResolve) {
19
- const resolve = pendingResolve;
20
- pendingResolve = null;
21
- resolve();
22
- }
37
+ wake();
38
+ };
39
+ // Release at once, even if nobody is reading; a quiet id would wait forever.
40
+ const stop = () => {
41
+ stopped = true;
42
+ unregisterLoggingStream(id, streamHandler);
43
+ logQueue.length = 0;
44
+ wake();
23
45
  };
24
46
  registerLoggingStream(id, streamHandler);
47
+ signal?.addEventListener('abort', stop, { once: true });
48
+ const removeCloseListener = scoped ? onSessionClose(sessionId, stop) : undefined;
25
49
  try {
26
50
  while (true) {
27
51
  while (logQueue.length > 0) {
28
52
  yield logQueue.shift();
29
53
  }
30
- // Wait for new logs - stream will terminate when client disconnects
54
+ // Stopped during a read: nothing to wake, so end here.
55
+ if (stopped || signal?.aborted)
56
+ return;
31
57
  await new Promise((resolve) => {
32
58
  pendingResolve = resolve;
33
59
  });
34
60
  }
35
61
  }
36
62
  finally {
63
+ signal?.removeEventListener('abort', stop);
64
+ removeCloseListener?.();
37
65
  unregisterLoggingStream(id, streamHandler);
38
66
  }
39
67
  }
@@ -1,7 +1,62 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
1
53
  import { getModelEntry } from '../runtime/model-registry.js';
2
54
  import { getPlugin, getPluginHandler } from '../plugins/index.js';
55
+ import { getRequestRegistry } from '../runtime/index.js';
56
+ import { OWNER_SESSION_ID } from '../runtime/session-id.js';
57
+ import { generateRandomRequestId } from '../runtime/request-id.js';
3
58
  import { profileReplyHandler, profileStreamHandler } from '../profiling/index.js';
4
- import { PluginNotFoundError, PluginHandlerNotFoundError, PluginHandlerTypeMismatchError, PluginRequestValidationFailedError, PluginResponseValidationFailedError, ModelNotFoundError } from '../errors/index.js';
59
+ import { InferenceCancelledError, PluginNotFoundError, PluginHandlerNotFoundError, PluginHandlerTypeMismatchError, PluginRequestValidationFailedError, PluginResponseValidationFailedError, ModelNotFoundError } from '../errors/index.js';
5
60
  import { getEngineLogger } from '../logging/index.js';
6
61
  import { formatZodError } from '../utils/zod-error.js';
7
62
  const logger = getEngineLogger();
@@ -22,57 +77,115 @@ function resolvePluginHandler(modelId, handlerName) {
22
77
  }
23
78
  return { modelType, plugin, handlerDef };
24
79
  }
80
+ // Every invoke is a registry request of kind `plugin` on the model, so a
81
+ // caller can cancel it by id and a session close or model unload sweeps it.
82
+ // The handler receives the abort signal through its context argument.
83
+ function beginInvoke(request) {
84
+ return getRequestRegistry().begin({
85
+ requestId: request.requestId ?? generateRandomRequestId(),
86
+ sessionId: request.sessionId,
87
+ kind: 'plugin',
88
+ modelId: request.modelId
89
+ });
90
+ }
91
+ // The owner's session is passed as undefined, the way the wire names it, so
92
+ // an inner request begun from the context registers like a direct one.
93
+ function invokeContext(ctx) {
94
+ return {
95
+ signal: ctx.signal,
96
+ requestId: ctx.requestId,
97
+ sessionId: ctx.sessionId === OWNER_SESSION_ID ? undefined : ctx.sessionId
98
+ };
99
+ }
25
100
  export async function handlePluginInvoke(request) {
26
101
  return profileReplyHandler({ op: 'pluginInvoke', request }, async () => {
27
- const { modelId, handler: handlerName, params } = request;
28
- logger.debug(`[pluginInvoke] modelId=${modelId} handler=${handlerName}`);
29
- const { handlerDef } = resolvePluginHandler(modelId, handlerName);
30
- if (handlerDef.streaming) {
31
- throw new PluginHandlerTypeMismatchError(handlerName, 'reply', 'streaming');
102
+ const env_1 = { stack: [], error: void 0, hasError: false };
103
+ try {
104
+ const { modelId, handler: handlerName, params } = request;
105
+ logger.debug(`[pluginInvoke] modelId=${modelId} handler=${handlerName}`);
106
+ const { handlerDef } = resolvePluginHandler(modelId, handlerName);
107
+ if (handlerDef.streaming) {
108
+ throw new PluginHandlerTypeMismatchError(handlerName, 'reply', 'streaming');
109
+ }
110
+ const parseResult = handlerDef.requestSchema.safeParse(params);
111
+ if (!parseResult.success) {
112
+ throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
113
+ }
114
+ const ctx = __addDisposableResource(env_1, await beginInvoke(request), true);
115
+ if (ctx.signal.aborted)
116
+ throw new InferenceCancelledError(ctx.requestId);
117
+ const result = await handlerDef.handler(parseResult.data, undefined, invokeContext(ctx));
118
+ if (ctx.signal.aborted)
119
+ throw new InferenceCancelledError(ctx.requestId);
120
+ const responseParseResult = handlerDef.responseSchema.safeParse(result);
121
+ if (!responseParseResult.success) {
122
+ throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
123
+ }
124
+ return {
125
+ type: 'pluginInvoke',
126
+ requestId: ctx.requestId,
127
+ result: responseParseResult.data
128
+ };
32
129
  }
33
- const parseResult = handlerDef.requestSchema.safeParse(params);
34
- if (!parseResult.success) {
35
- throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
130
+ catch (e_1) {
131
+ env_1.error = e_1;
132
+ env_1.hasError = true;
36
133
  }
37
- const result = await handlerDef.handler(parseResult.data);
38
- const responseParseResult = handlerDef.responseSchema.safeParse(result);
39
- if (!responseParseResult.success) {
40
- throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
134
+ finally {
135
+ const result_1 = __disposeResources(env_1);
136
+ if (result_1)
137
+ await result_1;
41
138
  }
42
- return {
43
- type: 'pluginInvoke',
44
- result: responseParseResult.data
45
- };
46
139
  });
47
140
  }
48
141
  export async function* handlePluginInvokeStream(request) {
49
142
  yield* profileStreamHandler({ op: 'pluginInvokeStream', request }, async function* () {
50
- const { modelId, handler: handlerName, params } = request;
51
- logger.debug(`[pluginInvokeStream] modelId=${modelId} handler=${handlerName}`);
52
- const { handlerDef } = resolvePluginHandler(modelId, handlerName);
53
- if (!handlerDef.streaming) {
54
- throw new PluginHandlerTypeMismatchError(handlerName, 'streaming', 'reply');
55
- }
56
- const parseResult = handlerDef.requestSchema.safeParse(params);
57
- if (!parseResult.success) {
58
- throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
59
- }
60
- const generator = handlerDef.handler(parseResult.data);
61
- for await (const chunk of generator) {
62
- const responseParseResult = handlerDef.responseSchema.safeParse(chunk);
63
- if (!responseParseResult.success) {
64
- throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
143
+ const env_2 = { stack: [], error: void 0, hasError: false };
144
+ try {
145
+ const { modelId, handler: handlerName, params } = request;
146
+ logger.debug(`[pluginInvokeStream] modelId=${modelId} handler=${handlerName}`);
147
+ const { handlerDef } = resolvePluginHandler(modelId, handlerName);
148
+ if (!handlerDef.streaming) {
149
+ throw new PluginHandlerTypeMismatchError(handlerName, 'streaming', 'reply');
150
+ }
151
+ const parseResult = handlerDef.requestSchema.safeParse(params);
152
+ if (!parseResult.success) {
153
+ throw new PluginRequestValidationFailedError(handlerName, formatZodError(parseResult.error));
65
154
  }
155
+ const ctx = __addDisposableResource(env_2, await beginInvoke(request), true);
156
+ if (ctx.signal.aborted)
157
+ throw new InferenceCancelledError(ctx.requestId);
158
+ const generator = handlerDef.handler(parseResult.data, undefined, invokeContext(ctx));
159
+ for await (const chunk of generator) {
160
+ if (ctx.signal.aborted)
161
+ throw new InferenceCancelledError(ctx.requestId);
162
+ const responseParseResult = handlerDef.responseSchema.safeParse(chunk);
163
+ if (!responseParseResult.success) {
164
+ throw new PluginResponseValidationFailedError(handlerName, formatZodError(responseParseResult.error));
165
+ }
166
+ yield {
167
+ type: 'pluginInvokeStream',
168
+ result: responseParseResult.data,
169
+ done: false
170
+ };
171
+ }
172
+ if (ctx.signal.aborted)
173
+ throw new InferenceCancelledError(ctx.requestId);
66
174
  yield {
67
175
  type: 'pluginInvokeStream',
68
- result: responseParseResult.data,
69
- done: false
176
+ requestId: ctx.requestId,
177
+ result: null,
178
+ done: true
70
179
  };
71
180
  }
72
- yield {
73
- type: 'pluginInvokeStream',
74
- result: null,
75
- done: true
76
- };
181
+ catch (e_2) {
182
+ env_2.error = e_2;
183
+ env_2.hasError = true;
184
+ }
185
+ finally {
186
+ const result_2 = __disposeResources(env_2);
187
+ if (result_2)
188
+ await result_2;
189
+ }
77
190
  });
78
191
  }
@@ -1,6 +1,6 @@
1
- import type { ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse, ModelRegistryEntry } from '../schemas/index.js';
1
+ import type { ModelRegistryListRequest, ModelRegistryListResponse, ModelRegistrySearchRequest, ModelRegistrySearchResponse, ModelRegistryGetModelRequest, ModelRegistryGetModelResponse, ModelRegistryEntry } from '../schemas/index.js';
2
2
  import type { QVACModelEntry } from '@qvac/registry-client';
3
3
  export declare function processRegistryModel(model: QVACModelEntry): ModelRegistryEntry | null;
4
- export declare function handleModelRegistryList(): Promise<ModelRegistryListResponse>;
4
+ export declare function handleModelRegistryList(request?: ModelRegistryListRequest): Promise<ModelRegistryListResponse>;
5
5
  export declare function handleModelRegistrySearch(request: ModelRegistrySearchRequest): Promise<ModelRegistrySearchResponse>;
6
6
  export declare function handleModelRegistryGetModel(request: ModelRegistryGetModelRequest): Promise<ModelRegistryGetModelResponse>;
@@ -1,9 +1,64 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
1
53
  import Buffer from 'bare-buffer';
2
54
  import { REGISTRY_ERROR_CODES } from '../schemas/errors.js';
3
55
  import { getAddonFromEngine, resolveCanonicalEngine, resolveRegistryEngine } from '../schemas/engine-addon-map.js';
4
- import { getRegistryClient } from '../runtime/registry-client.js';
56
+ import { acquireRegistryClient } from '../runtime/registry-client.js';
57
+ import { getRequestRegistry } from '../runtime/index.js';
58
+ import { generateRandomRequestId } from '../runtime/request-id.js';
59
+ import { resolveSessionId } from '../runtime/session-id.js';
5
60
  import { getEngineLogger } from '../logging/index.js';
6
- import { ModelRegistryQueryFailedError } from '../errors/index.js';
61
+ import { InferenceCancelledError, ModelRegistryBusyError, ModelRegistryModelNotFoundError, ModelRegistryQueryFailedError } from '../errors/index.js';
7
62
  const logger = getEngineLogger();
8
63
  function toHexString(value) {
9
64
  if (!value)
@@ -17,6 +72,46 @@ function toHexString(value) {
17
72
  }
18
73
  return '';
19
74
  }
75
+ // One query, cancellable by `requestId` while it waits for another process to
76
+ // release the registry.
77
+ async function withRegistryClient(request, query) {
78
+ const env_1 = { stack: [], error: void 0, hasError: false };
79
+ try {
80
+ const id = request.requestId ?? generateRandomRequestId();
81
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
82
+ requestId: id,
83
+ sessionId: resolveSessionId(request),
84
+ kind: 'modelRegistry'
85
+ }), true);
86
+ const lease = __addDisposableResource(env_1, await acquireRegistryClient({
87
+ wait: 'query',
88
+ signal: ctx.signal,
89
+ cancelled: () => new InferenceCancelledError(id)
90
+ })
91
+ // Awaited here: returning the promise would release the lease before the query ends.
92
+ , true);
93
+ // Awaited here: returning the promise would release the lease before the query ends.
94
+ return await query(lease.client);
95
+ }
96
+ catch (e_1) {
97
+ env_1.error = e_1;
98
+ env_1.hasError = true;
99
+ }
100
+ finally {
101
+ const result_1 = __disposeResources(env_1);
102
+ if (result_1)
103
+ await result_1;
104
+ }
105
+ }
106
+ // Errors a caller can act on pass through unwrapped.
107
+ function isPassThrough(error) {
108
+ const code = error.code;
109
+ return (error instanceof ModelRegistryBusyError ||
110
+ error instanceof InferenceCancelledError ||
111
+ code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
112
+ code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
113
+ code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND);
114
+ }
20
115
  export function processRegistryModel(model) {
21
116
  const engine = resolveRegistryEngine(model.engine || '', model.tags);
22
117
  if (!engine) {
@@ -52,11 +147,10 @@ export function processRegistryModel(model) {
52
147
  params: model.params || ''
53
148
  };
54
149
  }
55
- export async function handleModelRegistryList() {
150
+ export async function handleModelRegistryList(request = { type: 'modelRegistryList' }) {
56
151
  logger.debug('Handling QVAC model registry list request');
57
152
  try {
58
- const client = await getRegistryClient();
59
- const registryModels = await client.findBy();
153
+ const registryModels = await withRegistryClient(request, (client) => client.findBy());
60
154
  const models = registryModels
61
155
  .map(processRegistryModel)
62
156
  .filter((m) => m !== null);
@@ -69,13 +163,8 @@ export async function handleModelRegistryList() {
69
163
  }
70
164
  catch (error) {
71
165
  logger.error('QVAC model registry list failed:', error);
72
- // Re-throw registry client errors directly (19001-19003)
73
- const qvacError = error;
74
- if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
75
- qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
76
- qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
166
+ if (isPassThrough(error))
77
167
  throw error;
78
- }
79
168
  // Wrap unknown errors in a typed error
80
169
  throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
81
170
  }
@@ -83,10 +172,9 @@ export async function handleModelRegistryList() {
83
172
  export async function handleModelRegistrySearch(request) {
84
173
  logger.debug('Handling QVAC model registry search request', request);
85
174
  try {
86
- const client = await getRegistryClient();
87
- const registryModels = await client.findBy({
175
+ const registryModels = await withRegistryClient(request, (client) => client.findBy({
88
176
  ...(request.quantization && { quantization: request.quantization })
89
- });
177
+ }));
90
178
  let models = registryModels
91
179
  .map(processRegistryModel)
92
180
  .filter((m) => m !== null);
@@ -113,23 +201,17 @@ export async function handleModelRegistrySearch(request) {
113
201
  }
114
202
  catch (error) {
115
203
  logger.error('QVAC model registry search failed:', error);
116
- // Re-throw registry client errors directly (19001-19003)
117
- const qvacError = error;
118
- if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
119
- qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
120
- qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
204
+ if (isPassThrough(error))
121
205
  throw error;
122
- }
123
206
  throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
124
207
  }
125
208
  }
126
209
  export async function handleModelRegistryGetModel(request) {
127
210
  logger.debug('Handling QVAC model registry get model request', request);
128
211
  try {
129
- const client = await getRegistryClient();
130
- const rawModel = await client.getModel(request.registryPath, request.registrySource);
212
+ const rawModel = await withRegistryClient(request, (client) => client.getModel(request.registryPath, request.registrySource));
131
213
  if (!rawModel) {
132
- throw new ModelRegistryQueryFailedError(`Model not found: ${request.registrySource}/${request.registryPath}`);
214
+ throw new ModelRegistryModelNotFoundError(request.registrySource, request.registryPath);
133
215
  }
134
216
  const model = processRegistryModel(rawModel);
135
217
  if (!model) {
@@ -143,14 +225,11 @@ export async function handleModelRegistryGetModel(request) {
143
225
  };
144
226
  }
145
227
  catch (error) {
228
+ if (error instanceof ModelRegistryModelNotFoundError)
229
+ throw error;
146
230
  logger.error('QVAC model registry get model failed:', error);
147
- // Re-throw registry client errors directly (19001-19003)
148
- const qvacError = error;
149
- if (qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CONNECT ||
150
- qvacError.code === REGISTRY_ERROR_CODES.FAILED_TO_CLOSE ||
151
- qvacError.code === REGISTRY_ERROR_CODES.MODEL_NOT_FOUND) {
231
+ if (isPassThrough(error))
152
232
  throw error;
153
- }
154
233
  throw new ModelRegistryQueryFailedError(error instanceof Error ? error.message : 'Unknown error', error);
155
234
  }
156
235
  }
@@ -73,11 +73,11 @@ async function run(operation, work) {
73
73
  throw new RpcServerOperationError(operation, error instanceof Error ? error.message : String(error), error);
74
74
  }
75
75
  }
76
- async function withContext(kind, requestId, work) {
76
+ async function withContext(kind, requestId, sessionId, work) {
77
77
  const task = (async () => {
78
78
  const env_1 = { stack: [], error: void 0, hasError: false };
79
79
  try {
80
- const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({ requestId, kind }), true);
80
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({ requestId, sessionId, kind }), true);
81
81
  return await work(ctx);
82
82
  }
83
83
  catch (e_1) {
@@ -90,7 +90,7 @@ async function withContext(kind, requestId, work) {
90
90
  await result_1;
91
91
  }
92
92
  })();
93
- requests.set(task, requestId);
93
+ requests.set(task, { requestId, sessionId });
94
94
  try {
95
95
  return await task;
96
96
  }
@@ -105,7 +105,7 @@ async function withContext(kind, requestId, work) {
105
105
  }
106
106
  }
107
107
  export async function handleStartRpcServer(request) {
108
- return run('startRpcServer', () => withContext('rpcServer', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
108
+ return run('startRpcServer', () => withContext('rpcServer', request.requestId ?? generateRandomRequestId(), request.sessionId, async (ctx) => ({
109
109
  type: 'startRpcServer',
110
110
  ...(await rpcServers.start(request, ctx))
111
111
  })));
@@ -117,7 +117,7 @@ export async function handleStopRpcServer(request) {
117
117
  });
118
118
  }
119
119
  export async function handleDiscoverRpcServers(request) {
120
- return run('discoverRpcServers', () => withContext('rpcDiscovery', request.requestId ?? generateRandomRequestId(), async (ctx) => ({
120
+ return run('discoverRpcServers', () => withContext('rpcDiscovery', request.requestId ?? generateRandomRequestId(), request.sessionId, async (ctx) => ({
121
121
  type: 'discoverRpcServers',
122
122
  servers: await discoverRpcEndpoints(request.topic, request.timeoutMs ?? 5000, ctx)
123
123
  })));
@@ -126,8 +126,8 @@ export function closeRpcResources() {
126
126
  if (closing)
127
127
  return closing;
128
128
  closing = (async () => {
129
- for (const requestId of requests.values())
130
- getRequestRegistry().cancel({ requestId });
129
+ for (const target of requests.values())
130
+ getRequestRegistry().cancel(target);
131
131
  const stopping = rpcServers.close();
132
132
  const results = await Promise.allSettled([stopping, ...requests.keys()]);
133
133
  const stopResult = results[0];
@@ -0,0 +1,11 @@
1
+ import type { OpenSessionRequest, OpenSessionResponse, CloseSessionRequest, CloseSessionResponse } from '../schemas/session.js';
2
+ export declare function handleOpenSession(request: OpenSessionRequest): OpenSessionResponse;
3
+ /**
4
+ * Closing a session cancels its requests, waits for them to leave the
5
+ * registry, then disposes the vector indexes it opened and releases every
6
+ * model lease it holds. A model whose last lease was released is unloaded.
7
+ * The session record is removed first so a
8
+ * call racing the close is refused at dispatch instead of starting work on a
9
+ * session that is going away. Its id is not reusable until the close returns.
10
+ */
11
+ export declare function handleCloseSession(request: CloseSessionRequest): Promise<CloseSessionResponse>;
@@ -0,0 +1,40 @@
1
+ import { openSession, closeSession, finishSessionClose } from '../runtime/session-registry.js';
2
+ import { getRequestRegistry } from '../runtime/request-context.js';
3
+ import { getModelIdsLeasedBy } from '../runtime/model-registry.js';
4
+ import { disposeSessionVectorIndexes } from '../runtime/vector-index-registry.js';
5
+ import { unloadModel } from '../plugins/ops/unload-model.js';
6
+ import { getEngineLogger } from '../logging/index.js';
7
+ const logger = getEngineLogger();
8
+ export function handleOpenSession(request) {
9
+ const entry = openSession(request.sessionId, request.name);
10
+ return { type: 'openSession', success: true, sessionId: entry.id };
11
+ }
12
+ /**
13
+ * Closing a session cancels its requests, waits for them to leave the
14
+ * registry, then disposes the vector indexes it opened and releases every
15
+ * model lease it holds. A model whose last lease was released is unloaded.
16
+ * The session record is removed first so a
17
+ * call racing the close is refused at dispatch instead of starting work on a
18
+ * session that is going away. Its id is not reusable until the close returns.
19
+ */
20
+ export async function handleCloseSession(request) {
21
+ const { sessionId } = request;
22
+ closeSession(sessionId);
23
+ try {
24
+ const cancelled = await getRequestRegistry().drainSession(sessionId, 'sessionClosed');
25
+ disposeSessionVectorIndexes(sessionId);
26
+ const releasedModels = getModelIdsLeasedBy(sessionId);
27
+ for (const modelId of releasedModels) {
28
+ try {
29
+ await unloadModel({ modelId, sessionId });
30
+ }
31
+ catch (error) {
32
+ logger.error(`Error releasing model ${modelId} for session ${sessionId}:`, error);
33
+ }
34
+ }
35
+ return { type: 'closeSession', success: true, sessionId, cancelled, releasedModels };
36
+ }
37
+ finally {
38
+ finishSessionClose(sessionId);
39
+ }
40
+ }
@@ -1,4 +1,9 @@
1
- import type { Request, Response } from '../schemas/index.js';
1
+ import type { AbortSignalLike, Request, Response } from '../schemas/index.js';
2
+ /** Passed to a stream handler with `endsOnAbort`. */
3
+ export interface StreamHandlerContext {
4
+ /** Aborts when the caller stops reading. */
5
+ signal?: AbortSignalLike;
6
+ }
2
7
  export type ReplyHandler = (request: any, ...args: any[]) => Promise<Response> | Response;
3
8
  export type StreamHandler = (request: any, ...args: any[]) => AsyncGenerator<Response>;
4
9
  export type ProgressHandler = (request: any, ...args: any[]) => Promise<Response>;
@@ -8,4 +13,5 @@ export type HandlerEntry = {
8
13
  handler: ReplyHandler | StreamHandler | ProgressHandler | DuplexStreamHandler;
9
14
  supportsProgress?: boolean | ((request: Request) => boolean);
10
15
  pluginOp?: boolean;
16
+ endsOnAbort?: boolean;
11
17
  };