@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -1,8 +1,63 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import Buffer from 'bare-buffer';
1
54
  import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
2
55
  import { aggregateEvents } from '../utils/aggregate-events.js';
3
- import { CompletionFailedError } from '../errors/index.js';
4
- import { dispatchPluginStream } from './plugin-dispatch.js';
56
+ import { CompletionFailedError, InferenceCancelledError } from '../errors/index.js';
57
+ import { dispatchPluginStream } from '../handlers/plugin-dispatch.js';
5
58
  import { executeToolSearch, TOOL_SEARCH_NAME } from '../utils/tools/defer.js';
59
+ import { getRequestRegistry } from '../runtime/request-context.js';
60
+ import { generateRandomRequestId } from '../runtime/request-id.js';
6
61
  const DEFAULT_MAX_TOOL_TURNS = 8;
7
62
  // Generous ceiling on one client-side tool execution: long enough for a slow
8
63
  // MCP round trip, short enough that a client that stopped reading can't pin a
@@ -22,7 +77,8 @@ const MAX_TOOL_RESULT_LINE_BYTES = 16 * 1024 * 1024;
22
77
  * and the reader stops holding the request stream open.
23
78
  */
24
79
  export class ToolResultReader {
25
- buffer = '';
80
+ pending = [];
81
+ pendingBytes = 0;
26
82
  results = new Map();
27
83
  waiters = new Map();
28
84
  failed;
@@ -38,25 +94,42 @@ export class ToolResultReader {
38
94
  const next = await this.iterator.next();
39
95
  if (next.done)
40
96
  break;
41
- this.buffer += next.value.toString();
97
+ const chunk = next.value;
98
+ let start = 0;
42
99
  let nl;
43
- while ((nl = this.buffer.indexOf('\n')) !== -1) {
44
- const line = this.buffer.slice(0, nl);
45
- this.buffer = this.buffer.slice(nl + 1);
100
+ while ((nl = chunk.indexOf(0x0a, start)) !== -1) {
101
+ this.append(chunk.subarray(start, nl));
102
+ // A transport chunk can end inside a UTF-8 character. Decode complete lines only.
103
+ const line = Buffer.concat(this.pending).toString('utf8');
104
+ this.pending = [];
105
+ this.pendingBytes = 0;
46
106
  if (line.trim())
47
107
  this.accept(line);
108
+ start = nl + 1;
48
109
  }
49
- if (this.buffer.length > MAX_TOOL_RESULT_LINE_BYTES) {
50
- throw new CompletionFailedError('tool result frame exceeds the maximum size');
51
- }
110
+ this.append(chunk.subarray(start));
111
+ }
112
+ if (!this.closed) {
113
+ const tail = Buffer.concat(this.pending).toString('utf8');
114
+ this.pending = [];
115
+ this.pendingBytes = 0;
116
+ if (tail.trim())
117
+ this.accept(tail);
52
118
  }
53
- if (!this.closed && this.buffer.trim())
54
- this.accept(this.buffer);
55
119
  }
56
120
  catch (error) {
57
121
  this.failWaiters(error instanceof Error ? error : new Error(String(error)));
58
122
  }
59
123
  }
124
+ append(bytes) {
125
+ if (this.pendingBytes + bytes.length > MAX_TOOL_RESULT_LINE_BYTES) {
126
+ throw new CompletionFailedError('tool result frame exceeds the maximum size');
127
+ }
128
+ if (bytes.length) {
129
+ this.pending.push(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
130
+ this.pendingBytes += bytes.length;
131
+ }
132
+ }
60
133
  failWaiters(error) {
61
134
  this.failed = error;
62
135
  // Wake every waiter; each rethrows via waitFor's failure check.
@@ -76,7 +149,12 @@ export class ToolResultReader {
76
149
  this.results.set(parsed.callId, parsed);
77
150
  }
78
151
  }
79
- async waitFor(callId, timeoutMs = TOOL_RESULT_TIMEOUT_MS) {
152
+ /** Rejects with `InferenceCancelledError` when `signal` aborts before the result arrives. */
153
+ async waitFor(callId, timeoutMs = TOOL_RESULT_TIMEOUT_MS, signal) {
154
+ // A cancel that landed between turns aborts the signal before the wait
155
+ // starts; it wins over a result the client already sent.
156
+ if (signal?.aborted)
157
+ throw new InferenceCancelledError(callId);
80
158
  const buffered = this.results.get(callId);
81
159
  if (buffered) {
82
160
  this.results.delete(callId);
@@ -85,12 +163,22 @@ export class ToolResultReader {
85
163
  if (this.failed)
86
164
  throw this.failed;
87
165
  return await new Promise((resolve, reject) => {
88
- const timer = setTimeout(() => {
166
+ const settle = () => {
167
+ clearTimeout(timer);
168
+ signal?.removeEventListener('abort', onAbort);
89
169
  this.waiters.delete(callId);
170
+ };
171
+ const onAbort = () => {
172
+ settle();
173
+ reject(new InferenceCancelledError(callId));
174
+ };
175
+ const timer = setTimeout(() => {
176
+ settle();
90
177
  reject(new CompletionFailedError(`tool callback ${callId} timed out`));
91
178
  }, timeoutMs);
179
+ signal?.addEventListener('abort', onAbort, { once: true });
92
180
  this.waiters.set(callId, (result) => {
93
- clearTimeout(timer);
181
+ settle();
94
182
  resolve(result);
95
183
  });
96
184
  });
@@ -104,7 +192,8 @@ export class ToolResultReader {
104
192
  if (this.closed)
105
193
  return;
106
194
  this.closed = true;
107
- this.buffer = '';
195
+ this.pending = [];
196
+ this.pendingBytes = 0;
108
197
  this.results.clear();
109
198
  if (!this.failed)
110
199
  this.failWaiters(new CompletionFailedError('tool result stream closed'));
@@ -125,10 +214,11 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
125
214
  // the turn that's currently generating — the expensive part a Stop button
126
215
  // targets. Turns run strictly sequentially: turn N's request-registry entry
127
216
  // is disposed (on the `for await` break below) before turn N+1 calls
128
- // `begin(...)`, so reusing the id can't collide in `entries`. A cancel that
129
- // lands between turns (during a tool-result wait) is caught by the
130
- // registry's cancel-before-begin tripwire and aborts the next turn's
131
- // `begin(...)`. Each turn's `history` and `type` are set below.
217
+ // `begin(...)`, so reusing the id can't collide in `entries`. The handler
218
+ // registers each tool-result wait under the same id (see
219
+ // `registeredToolWaits`), so a cancel during the wait ends it; one that
220
+ // lands between registrations is caught by the registry's
221
+ // cancel-before-begin tripwire. Each turn's `history` and `type` are set below.
132
222
  const { maxToolTurns, ...turnBase } = request;
133
223
  const maxTurns = maxToolTurns ?? DEFAULT_MAX_TOOL_TURNS;
134
224
  let history = [...request.history];
@@ -159,7 +249,7 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
159
249
  const defers = request.tools?.some((tool) => tool.deferLoading === true) ?? false;
160
250
  for (const call of toolCalls) {
161
251
  // `tool_search` is the SDK's own tool: it reads the registered inventory
162
- // and appends the matched definitions, so it never leaves the worker and
252
+ // and records what it loaded in the history, so it never leaves the worker and
163
253
  // the client is never asked to run it. The `toolCall` event still went
164
254
  // downstream with the turn's events, so the client can see it happened.
165
255
  if (defers && call.name === TOOL_SEARCH_NAME) {
@@ -167,7 +257,7 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
167
257
  ...history,
168
258
  {
169
259
  role: 'tool',
170
- content: executeToolSearch(request.tools ?? [], call.arguments, history)
260
+ content: executeToolSearch(request, call.arguments, history)
171
261
  }
172
262
  ];
173
263
  continue;
@@ -177,7 +267,23 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
177
267
  turn,
178
268
  toolCallback: { callId: call.id, name: call.name, arguments: call.arguments }
179
269
  };
180
- const result = await reader.waitFor(call.id);
270
+ let result;
271
+ try {
272
+ result = await reader.waitFor(call.id);
273
+ }
274
+ catch (error) {
275
+ if (!(error instanceof InferenceCancelledError))
276
+ throw error;
277
+ // Cancelled while the client ran its tool: end the way a cancelled
278
+ // turn ends, so the client folds a `cancelled` stop.
279
+ yield {
280
+ type: 'completionOrchestrate',
281
+ turn: turn + 1,
282
+ events: [{ type: 'completionDone', seq: 0, stopReason: 'cancelled' }]
283
+ };
284
+ yield { type: 'completionOrchestrate', done: true };
285
+ return;
286
+ }
181
287
  const content = result.error !== undefined
182
288
  ? JSON.stringify({ error: result.error })
183
289
  : JSON.stringify(result.result ?? null);
@@ -190,13 +296,49 @@ export async function* orchestrateCompletion(request, runTurn, reader) {
190
296
  // whatever the model produced.
191
297
  yield { type: 'completionOrchestrate', done: true, stopReason: 'maxToolTurns' };
192
298
  }
299
+ /**
300
+ * Waits for a tool result as a request of its own, under the completion's
301
+ * `(sessionId, requestId)`, so the registry sees the completion between its
302
+ * turns: a cancel by id or by model, the transport signal and a session close
303
+ * stop the wait, and a session close waits for the completion to end.
304
+ */
305
+ export function registeredToolWaits(reader, request) {
306
+ const requestId = request.requestId ?? generateRandomRequestId();
307
+ return {
308
+ async waitFor(callId) {
309
+ const env_1 = { stack: [], error: void 0, hasError: false };
310
+ try {
311
+ // The model id lets `cancel({ modelId })` and an unload reach the wait.
312
+ // An unbounded cap skips slot admission, so the wait holds none of the
313
+ // model's completion slots while the client runs its tool.
314
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
315
+ requestId,
316
+ sessionId: request.sessionId,
317
+ kind: 'completion',
318
+ modelId: request.modelId,
319
+ maxConcurrentPerModel: Infinity
320
+ }), true);
321
+ return await reader.waitFor(callId, undefined, ctx.signal);
322
+ }
323
+ catch (e_1) {
324
+ env_1.error = e_1;
325
+ env_1.hasError = true;
326
+ }
327
+ finally {
328
+ const result_1 = __disposeResources(env_1);
329
+ if (result_1)
330
+ await result_1;
331
+ }
332
+ }
333
+ };
334
+ }
193
335
  export async function* handleCompletionOrchestrate(request, inputStream) {
194
336
  const reader = new ToolResultReader(inputStream);
195
337
  const runTurn = async function* (innerRequest) {
196
338
  yield* dispatchPluginStream(innerRequest.modelId, 'completionStream', innerRequest);
197
339
  };
198
340
  try {
199
- yield* orchestrateCompletion(request, runTurn, reader);
341
+ yield* orchestrateCompletion(request, runTurn, registeredToolWaits(reader, request));
200
342
  }
201
343
  finally {
202
344
  // Tear the reader down however orchestration ends -- natural finish, turn
@@ -1,6 +1,7 @@
1
1
  import type { DeleteCacheRequest, DeleteCacheResponse } from '../schemas/index.js';
2
2
  /**
3
- * RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
3
+ * RPC handler for `deleteCache(...)`. `modelSrc` deletes a registry model's
4
+ * cached download instead of KV caches. The three KV-cache bookkeeping
4
5
  * layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
5
6
  * are private to `kv-cache-session.ts`; this handler delegates to that
6
7
  * module's single administrative entry point (`deleteKvCacheState`).
@@ -1,8 +1,10 @@
1
1
  import { deleteKvCacheState } from '../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
2
+ import { deleteRegistryModelCache } from '../handlers/load-model/registry-delete.js';
2
3
  import { getEngineLogger } from '../logging/index.js';
3
4
  const logger = getEngineLogger();
4
5
  /**
5
- * RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
6
+ * RPC handler for `deleteCache(...)`. `modelSrc` deletes a registry model's
7
+ * cached download instead of KV caches. The three KV-cache bookkeeping
6
8
  * layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
7
9
  * are private to `kv-cache-session.ts`; this handler delegates to that
8
10
  * module's single administrative entry point (`deleteKvCacheState`).
@@ -18,6 +20,9 @@ export async function handleDeleteCache(request) {
18
20
  else if ('auto' in request && request.auto) {
19
21
  await deleteKvCacheState({ auto: true });
20
22
  }
23
+ else if ('modelSrc' in request) {
24
+ await deleteRegistryModelCache(request.modelSrc);
25
+ }
21
26
  else if ('kvCacheKey' in request) {
22
27
  await deleteKvCacheState({
23
28
  kvCacheKey: request.kvCacheKey,
@@ -51,13 +51,14 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
53
  import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
54
- import { resolveModelPath, resolveModelPathWithStats } from './load-model/resolve.js';
55
- import { buildDownloadProfilingFields } from './load-model/types.js';
54
+ import { resolveModelPath, resolveModelPathWithStats } from '../handlers/load-model/resolve.js';
55
+ import { buildDownloadProfilingFields } from '../handlers/load-model/types.js';
56
56
  import { nowMs, generateProfileId } from '../profiling/clock.js';
57
57
  import { getEngineLogger } from '../logging/index.js';
58
58
  import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
59
59
  import { generateRandomRequestId } from '../runtime/request-id.js';
60
- import { InferenceCancelledError } from '../errors/index.js';
60
+ import { resolveSessionId } from '../runtime/session-id.js';
61
+ import { InferenceCancelledError, ModelRegistryBusyError } from '../errors/index.js';
61
62
  const logger = getEngineLogger();
62
63
  export async function handleDownloadAsset(request, progressCallback) {
63
64
  const env_1 = { stack: [], error: void 0, hasError: false };
@@ -67,12 +68,14 @@ export async function handleDownloadAsset(request, progressCallback) {
67
68
  const profilingMeta = request[PROFILING_KEY];
68
69
  const profilingEnabled = profilingMeta?.enabled !== false && !!profilingMeta;
69
70
  const requestId = request.requestId ?? generateRandomRequestId();
71
+ const sessionId = resolveSessionId(request);
70
72
  // `downloadAsset` is artifact-shaped, not model-shaped — there is no
71
73
  // `modelId` to register on the registry entry. Cancel by `requestId`
72
74
  // is the primary path; `cancel({ modelId })` is intentionally a
73
75
  // non-match for this kind.
74
76
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
75
77
  requestId,
78
+ sessionId,
76
79
  kind: 'downloadAsset'
77
80
  }), true);
78
81
  const log = withRequestContext(getEngineLogger(), ctx);
@@ -81,7 +84,8 @@ export async function handleDownloadAsset(request, progressCallback) {
81
84
  requestBinding: {
82
85
  signal: ctx.signal,
83
86
  scope: ctx.scope,
84
- requestId
87
+ requestId,
88
+ sessionId
85
89
  }
86
90
  };
87
91
  try {
@@ -121,12 +125,15 @@ export async function handleDownloadAsset(request, progressCallback) {
121
125
  catch (error) {
122
126
  // Mirror the load-model handler's cancel contract: a typed cancel
123
127
  // bubbles up as the rejection so client-side callers can branch on
124
- // `instanceof InferenceCancelledError`. Every other error keeps
125
- // the legacy `success: false` envelope.
128
+ // `instanceof InferenceCancelledError`, and so does a busy registry.
129
+ // Every other error keeps the legacy `success: false` envelope.
126
130
  if (error instanceof InferenceCancelledError) {
127
131
  log.info(`downloadAsset cancelled requestId=${requestId}`);
128
132
  throw error;
129
133
  }
134
+ // Typed so callers can tell another process has the registry.
135
+ if (error instanceof ModelRegistryBusyError)
136
+ throw error;
130
137
  logger.error('Error downloading asset:', error);
131
138
  return {
132
139
  type: 'downloadAsset',
@@ -1,4 +1,4 @@
1
- import { dispatchPluginReply } from './plugin-dispatch.js';
1
+ import { dispatchPluginReply } from '../handlers/plugin-dispatch.js';
2
2
  import { startFinetune } from '../plugins/builtin/llamacpp-completion/ops/finetune.js';
3
3
  export async function handleFinetune(request, progressCallback) {
4
4
  if (progressCallback &&
@@ -1,6 +1,6 @@
1
1
  import { getModelEntry } from '../runtime/model-registry.js';
2
2
  import { getPlugin } from '../plugins/registry.js';
3
- import { detectToolDialectFromName } from '../utils/tools/index.js';
3
+ import { detectToolDialect } from '../utils/tool-integration.js';
4
4
  import { ModelNotFoundError } from '../errors/index.js';
5
5
  import { getEngineLogger } from '../logging/index.js';
6
6
  const logger = getEngineLogger();
@@ -21,13 +21,13 @@ export function handleGetLoadedModelInfo(request) {
21
21
  modelType: entry.local.modelType,
22
22
  handlers,
23
23
  loadedAt: entry.local.loadedAt,
24
+ leaseCount: entry.leases.size,
24
25
  ...(plugin && { displayName: plugin.displayName }),
25
26
  ...(plugin && { addonPackage: plugin.addonPackage }),
26
27
  ...(entry.local.name && { name: entry.local.name }),
27
28
  ...(entry.local.path && { path: entry.local.path }),
28
- // Same detection the completion normalizer uses, so callers see the dialect it parses.
29
29
  ...(handlers.includes('completionStream') && {
30
- toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
30
+ toolDialect: detectToolDialect(modelId)
31
31
  }),
32
32
  ...(fitProbe !== undefined && { fitProbe })
33
33
  };
@@ -1,3 +1,55 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
1
53
  import { models } from '../models/registry/models.js';
2
54
  import { getAllModelIds, getModelEntry } from '../runtime/model-registry.js';
3
55
  import { generateShortHash } from '../utils/index.js';
@@ -5,8 +57,8 @@ import { promises as fsPromises } from 'bare-fs';
5
57
  import { getShardPath, getModelsCacheDir, getSingleFileCachePath } from '../utils/cache/paths.js';
6
58
  import { validateAndJoinPath } from '../utils/path-security.js';
7
59
  import { ModelNotFoundError } from '../errors/index.js';
8
- import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from './load-model/coreml-sidecars.js';
9
- import { getRegistryClient } from '../runtime/registry-client.js';
60
+ import { findCoremlCompanionSet, findLocallyCachedCoremlCompanionSet, getCoremlSidecarSpec, getCoremlSetKey } from '../handlers/load-model/coreml-sidecars.js';
61
+ import { acquireRegistryClient } from '../runtime/registry-client.js';
10
62
  import { getRuntimeContext } from '../runtime/state.js';
11
63
  export async function handleGetModelInfo(request) {
12
64
  const { name } = request;
@@ -81,7 +133,24 @@ async function handleOptionalCoremlModel(model) {
81
133
  const filename = model.registryPath.split('/').pop() || model.registryPath;
82
134
  const primaryPath = validateAndJoinPath(getModelsCacheDir(), 'sets', getCoremlSetKey(model, spec), filename);
83
135
  await fsPromises.access(primaryPath);
84
- const set = await findCoremlCompanionSet(await getRegistryClient(), model, platform);
136
+ let set;
137
+ {
138
+ const env_1 = { stack: [], error: void 0, hasError: false };
139
+ try {
140
+ // A status check reports what is on disk rather than wait on another process.
141
+ const lease = __addDisposableResource(env_1, await acquireRegistryClient({ wait: 'status' }), true);
142
+ set = await findCoremlCompanionSet(lease.client, model, platform);
143
+ }
144
+ catch (e_1) {
145
+ env_1.error = e_1;
146
+ env_1.hasError = true;
147
+ }
148
+ finally {
149
+ const result_1 = __disposeResources(env_1);
150
+ if (result_1)
151
+ await result_1;
152
+ }
153
+ }
85
154
  if (set) {
86
155
  const status = await handleCompanionSetModel(set, model.registryPath, model.addon);
87
156
  if (status.isCached)
@@ -1,16 +1,21 @@
1
1
  import type { QVACRegistryClient } from '@qvac/registry-client';
2
2
  import type { RegistryItem } from '../../models/registry/models.js';
3
- type BundleSpec = {
4
- sourceName: string;
5
- targetSuffix: string;
6
- };
7
- export type CoremlSidecarSpec = {
8
- root: string;
9
- modelStem: string;
10
- bundles: readonly BundleSpec[];
11
- };
3
+ import { getCoremlSidecarSpec, type CoremlSidecarSpec } from '../../utils/cache/layout.js';
4
+ export { getCoremlSidecarSpec, type CoremlSidecarSpec };
12
5
  type CompanionSet = NonNullable<RegistryItem['companionSet']>;
13
- export declare function getCoremlSidecarSpec(registryPath: string, registrySource: string, platform: string | undefined): CoremlSidecarSpec | undefined;
6
+ /**
7
+ * For the encoder-less GGUF of a Core ML distribution, the model constant to
8
+ * load instead where its bundles cannot be staged. The loader stages bundles
9
+ * on macOS and iOS alone, so anywhere else such a load is refused up front
10
+ * rather than failing inside the addon after the download.
11
+ */
12
+ export declare function getCoremlOnlyAlternative(registryPath: string, registrySource: string): string | undefined;
13
+ /**
14
+ * Why a `registry://` model source cannot load on `platform`, or `undefined`
15
+ * when it can. Only the encoder-less GGUF of a Core ML distribution is refused,
16
+ * and only off Apple platforms; every other source is the loader's business.
17
+ */
18
+ export declare function getRegistryModelPlatformRefusal(modelSrc: unknown, platform: string | undefined): string | undefined;
14
19
  export declare function getCoremlSetKey(model: RegistryItem, spec: CoremlSidecarSpec): string;
15
20
  export declare function readCachedCoremlCompanionSet(model: RegistryItem, platform: string | undefined, cacheDir?: string): Promise<CompanionSet | undefined>;
16
21
  export declare function writeCachedCoremlCompanionSet(set: CompanionSet, cacheDir?: string): Promise<void>;
@@ -22,4 +27,3 @@ export declare function findLocallyCachedCoremlCompanionSet(model: RegistryItem,
22
27
  * this lets the SDK use them as soon as the registry has ingested the files.
23
28
  */
24
29
  export declare function findCoremlCompanionSet(client: Pick<QVACRegistryClient, 'findModels'>, model: RegistryItem, platform: string | undefined): Promise<CompanionSet | undefined>;
25
- export {};