@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -1,5 +1,5 @@
1
1
  import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo, resolveCacheFilePath } from '../../../../plugins/ops/kv-cache-utils.js';
3
3
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
4
  import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
5
  import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
@@ -9,7 +9,7 @@ import { parseToolCalls } from '../../../../utils/tools/index.js';
9
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
10
10
  import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
11
11
  import { getEngineLogger } from '../../../../logging/index.js';
12
- import { AttachmentNotFoundError } from '../../../../errors/index.js';
12
+ import { AttachmentNotFoundError, InvalidSeedFromParamsError, KvCacheSeedSharingDisabledError, KvCacheSeedSourceNotFoundError, RequestValidationFailedError } from '../../../../errors/index.js';
13
13
  import { nowMs } from '../../../../profiling/index.js';
14
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
15
15
  import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
@@ -34,6 +34,21 @@ function transformMessage(message) {
34
34
  return transformed;
35
35
  }
36
36
  const msg = message;
37
+ // Ordered parts reach the addon as ONE message, so text and each image share
38
+ // a turn. Attachments cannot express that: they are all sent before the
39
+ // text, which leaves the model unable to tell which image is which option.
40
+ if (Array.isArray(msg.content)) {
41
+ const parts = msg.content.map((part) => {
42
+ if (part.type === 'text')
43
+ return { type: 'text', text: part.text };
44
+ if (!fs.existsSync(part.path)) {
45
+ throw new AttachmentNotFoundError(part.path);
46
+ }
47
+ return { type: 'media', content: part.path };
48
+ });
49
+ transformed.push({ role: msg.role, content: parts });
50
+ return transformed;
51
+ }
37
52
  if (msg.attachments && msg.attachments.length > 0) {
38
53
  for (const attachment of msg.attachments) {
39
54
  if (!fs.existsSync(attachment.path)) {
@@ -143,6 +158,9 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
143
158
  }),
144
159
  ...(cacheOptions?.prefill !== undefined && {
145
160
  prefill: cacheOptions.prefill
161
+ }),
162
+ ...(cacheOptions?.baseCacheKey !== undefined && {
163
+ baseCacheKey: cacheOptions.baseCacheKey
146
164
  })
147
165
  };
148
166
  const hasRunOptions = Object.keys(runOptions).length > 0;
@@ -176,20 +194,31 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
176
194
  toolCalls: toolCallsResult,
177
195
  responseText: accumulatedText,
178
196
  producedTokens,
197
+ ...(stopReason === 'eos' && { stopReason: 'eos' }),
198
+ ...(stopReason === 'antiprompt' && { stopReason: 'stopSequence' }),
179
199
  stoppedAtContextBoundary: stopReason === 'contextOverflow'
180
200
  };
181
201
  }
182
202
  export async function* completion(params, opts) {
183
- const { history: requestHistory, modelId, kvCache, prefill, tools, generationParams, responseFormat } = params;
203
+ const { history: requestHistory, modelId, kvCache, prefill, seedFrom, tools, toolSearch, generationParams, responseFormat } = params;
184
204
  const { signal, scope } = opts;
185
205
  const requestLogger = opts.logger ?? logger;
206
+ // seedFrom needs a named kvCache key other than itself: the auto key is
207
+ // never returned to the caller, so it cannot be named on either side.
208
+ if (seedFrom !== undefined && (typeof kvCache !== 'string' || seedFrom === kvCache)) {
209
+ throw new InvalidSeedFromParamsError();
210
+ }
186
211
  const modelConfig = getModelConfig(modelId);
187
212
  const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
213
+ // The addon cannot render a prompt without messages; tools alone are not enough.
214
+ if (prefill && history.length === 0) {
215
+ throw new RequestValidationFailedError('prefill needs at least one message: add a system message to history or set modelConfig.system_prompt');
216
+ }
188
217
  const toolsEnabled = modelConfig.tools === true;
189
218
  // With deferred tools the prompt carries the always-loaded ones, `tool_search`
190
219
  // and its catalog, and whatever earlier searches loaded. `null` when nothing
191
220
  // defers, which leaves the existing path alone.
192
- const deferred = resolveDeferredTools(tools, history);
221
+ const deferred = resolveDeferredTools(tools, history, toolSearch);
193
222
  const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
194
223
  const toolsActive = !!toolsToRender?.length && toolsEnabled;
195
224
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
@@ -288,8 +317,35 @@ export async function* completion(params, opts) {
288
317
  // defer above so this runs before the listener detach.
289
318
  let rollbackOnUnwind = false;
290
319
  scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
320
+ // A seed starts a chat that has no cache yet; a chat that has one keeps it.
321
+ // The addon ignores `baseCacheKey` on a model loaded with sharing off and
322
+ // skips a missing source silently, so both are checked here first.
323
+ let baseCacheKey;
324
+ if (seedFrom !== undefined && turn.createdCache) {
325
+ if (modelConfig.cache_share_across_keys !== true) {
326
+ throw new KvCacheSeedSharingDisabledError(modelId);
327
+ }
328
+ const sourcePath = resolveCacheFilePath(modelId, configHash, seedFrom);
329
+ try {
330
+ await fs.promises.access(sourcePath);
331
+ }
332
+ catch (error) {
333
+ // Only a missing file is "not found"; a permission or I/O error keeps its cause.
334
+ if (error.code !== 'ENOENT')
335
+ throw error;
336
+ throw new KvCacheSeedSourceNotFoundError(seedFrom, modelId);
337
+ }
338
+ baseCacheKey = sourcePath;
339
+ }
340
+ else if (seedFrom !== undefined) {
341
+ requestLogger.debug(`[kv-cache] seedFrom ignored: "${kvCache}" already has a cache`);
342
+ }
291
343
  logMessagesToAddon(prompt, 'PROMPT_SEND');
292
- const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, { cacheKey: turn.cachePath, ...(prefill !== undefined && { prefill }) }, dialect, setActiveResponse);
344
+ const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, {
345
+ cacheKey: turn.cachePath,
346
+ ...(prefill !== undefined && { prefill }),
347
+ ...(baseCacheKey !== undefined && { baseCacheKey })
348
+ }, dialect, setActiveResponse);
293
349
  if (typeof kvCache === 'string') {
294
350
  // Custom-key path: the file holds whatever the addon kept, cancelled and
295
351
  // stopped turns included.
@@ -1,10 +1,11 @@
1
+ import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
1
2
  import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from '../../../../schemas/index.js';
2
3
  type FinetuneOptions = FinetuneRunParams['options'];
3
4
  export declare function clearFinetuneRuntimeState(modelId: string): void;
4
5
  export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
5
- export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
6
- export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
7
- export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
6
+ export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
7
+ export declare function pauseFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
8
+ export declare function cancelFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
8
9
  export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
9
- export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
10
+ export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
10
11
  export {};
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../../../schemas/plugin.js';
53
54
  import fs from 'bare-fs';
54
55
  import { getModel } from '../../../../runtime/model-registry.js';
55
56
  import { CompletionFailedError } from '../../../../errors/index.js';
56
57
  import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
57
58
  import { generateRandomRequestId } from '../../../../runtime/request-id.js';
59
+ import { OWNER_SESSION_ID, resolveSessionId } from '../../../../runtime/session-id.js';
58
60
  import { getEngineLogger } from '../../../../logging/index.js';
59
61
  const PAUSE_CHECKPOINT_PREFIX = 'pause_checkpoint_step_';
60
62
  // Ref-count, not a Set: with an exclusive writer a second finetune queues behind
@@ -94,15 +96,33 @@ const finetuneRequestIds = new Map();
94
96
  // not settled. A new finetune is rejected during this window even if the runtime ref
95
97
  // has already cleared, because handle.await() can settle before model.pause() resolves.
96
98
  const pausingModels = new Set();
99
+ // A finetune may be stopped by the session that started it, or by the owner,
100
+ // who may stop any session's finetune. Another session's pause or cancel is a
101
+ // no-op, as its registry cancel would be.
102
+ function callerMayStop(tracked, callerSessionId) {
103
+ if (callerSessionId === OWNER_SESSION_ID)
104
+ return true;
105
+ return (tracked.sessionId ?? OWNER_SESSION_ID) === callerSessionId;
106
+ }
97
107
  // Cancel the model's in-flight finetune by its tracked requestId (covers the
98
108
  // registry grant→register handoff window via the cancel-before-begin tripwire),
99
- // falling back to a broad cancel if the id is unknown.
100
- function cancelTrackedFinetune(modelId) {
101
- const requestId = finetuneRequestIds.get(modelId);
102
- if (requestId !== undefined)
103
- getRequestRegistry().cancel({ requestId });
104
- else
105
- getRequestRegistry().cancel({ modelId, kind: 'finetune' });
109
+ // falling back to a broad cancel within the caller's own session if the id is
110
+ // unknown. Returns false when the finetune belongs to another session.
111
+ function cancelTrackedFinetune(modelId, callerSessionId) {
112
+ const tracked = finetuneRequestIds.get(modelId);
113
+ if (tracked !== undefined) {
114
+ if (!callerMayStop(tracked, callerSessionId))
115
+ return false;
116
+ getRequestRegistry().cancel(tracked);
117
+ return true;
118
+ }
119
+ if (callerSessionId === OWNER_SESSION_ID) {
120
+ getRequestRegistry().cancel({ modelId, kind: 'finetune', allSessions: true });
121
+ }
122
+ else {
123
+ getRequestRegistry().cancel({ modelId, kind: 'finetune', sessionId: callerSessionId });
124
+ }
125
+ return true;
106
126
  }
107
127
  export function getFinetuneStateFromCheckpoints(options) {
108
128
  const checkpointDirectory = options.checkpointSaveDir ?? './checkpoints';
@@ -137,7 +157,7 @@ function validateExplicitFinetuneOperation(request) {
137
157
  throw new CompletionFailedError(`Model "${request.modelId}" has no paused finetune checkpoint to resume`);
138
158
  }
139
159
  }
140
- export async function startFinetune(request, onProgress) {
160
+ export async function startFinetune(request, onProgress, context) {
141
161
  const env_1 = { stack: [], error: void 0, hasError: false };
142
162
  try {
143
163
  const model = getModel(request.modelId);
@@ -156,19 +176,25 @@ export async function startFinetune(request, onProgress) {
156
176
  // observes RUNNING, not IDLE, and a concurrent start is rejected above. Released
157
177
  // on a failed begin() or on scope unwind below.
158
178
  registerRunningFinetune(request.modelId);
159
- finetuneRequestIds.set(request.modelId, requestId);
179
+ // Under an invoke the job runs as the invoke's session, the owner included,
180
+ // so the stop paths compare against that session and not one named inside
181
+ // the params.
182
+ const tracked = { requestId, sessionId: context ? context.sessionId : request.sessionId };
183
+ finetuneRequestIds.set(request.modelId, tracked);
160
184
  // Scope the run into the registry so cancel({ requestId }) and
161
185
  // cancel({ modelId, kind: "finetune" }) reach it; onAbort forwards to
162
186
  // model.cancel().
163
187
  const ctx = __addDisposableResource(env_1, await getRequestRegistry()
164
188
  .begin({
165
189
  requestId,
190
+ sessionId: request.sessionId,
191
+ ...requestParent(context),
166
192
  kind: 'finetune',
167
193
  modelId: request.modelId
168
194
  })
169
195
  .catch((err) => {
170
196
  clearFinetuneRuntimeState(request.modelId);
171
- if (finetuneRequestIds.get(request.modelId) === requestId) {
197
+ if (finetuneRequestIds.get(request.modelId) === tracked) {
172
198
  finetuneRequestIds.delete(request.modelId);
173
199
  }
174
200
  throw err;
@@ -178,7 +204,7 @@ export async function startFinetune(request, onProgress) {
178
204
  // removes the listener first.
179
205
  ctx.scope.defer(() => {
180
206
  clearFinetuneRuntimeState(request.modelId);
181
- if (finetuneRequestIds.get(request.modelId) === requestId) {
207
+ if (finetuneRequestIds.get(request.modelId) === tracked) {
182
208
  finetuneRequestIds.delete(request.modelId);
183
209
  }
184
210
  });
@@ -237,10 +263,14 @@ export async function startFinetune(request, onProgress) {
237
263
  await result_1;
238
264
  }
239
265
  }
240
- export async function pauseFinetune(modelId) {
266
+ export async function pauseFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
241
267
  // Validate the model is loaded on every path (matches main; getModel throws
242
268
  // ModelNotFoundError for an unknown/unloaded model) before reporting any state.
243
269
  const model = getModel(modelId);
270
+ const tracked = finetuneRequestIds.get(modelId);
271
+ if (tracked !== undefined && !callerMayStop(tracked, callerSessionId)) {
272
+ return { type: 'finetune', status: 'RUNNING' };
273
+ }
244
274
  if (admittedFinetunes.has(modelId)) {
245
275
  // The single finetune holds the model exclusively, so the addon-global pause is
246
276
  // safe (nothing else runs on the model). Bar a new finetune until the native
@@ -260,7 +290,7 @@ export async function pauseFinetune(modelId) {
260
290
  // pause would cancel that unrelated reader, so cancel the queued finetune by its
261
291
  // own requestId. Nothing has trained, so report CANCELLED rather than a false
262
292
  // PAUSED.
263
- cancelTrackedFinetune(modelId);
293
+ cancelTrackedFinetune(modelId, callerSessionId);
264
294
  return { type: 'finetune', status: 'CANCELLED' };
265
295
  }
266
296
  // No finetune on this model: a no-op pause, but the model was validated above.
@@ -268,8 +298,10 @@ export async function pauseFinetune(modelId) {
268
298
  }
269
299
  // Routes cancellation through the registry; the model.cancel() forward is
270
300
  // installed by startFinetune, so never call model.cancel() here.
271
- export function cancelFinetune(modelId) {
272
- cancelTrackedFinetune(modelId);
301
+ export function cancelFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
302
+ if (!cancelTrackedFinetune(modelId, callerSessionId)) {
303
+ return Promise.resolve({ type: 'finetune', status: 'RUNNING' });
304
+ }
273
305
  return Promise.resolve({
274
306
  type: 'finetune',
275
307
  status: 'CANCELLED'
@@ -282,18 +314,21 @@ export function getFinetuneState(params) {
282
314
  status: runtimeState ? 'RUNNING' : getFinetuneStateFromCheckpoints(params.options)
283
315
  };
284
316
  }
285
- export async function finetune(request, onProgress) {
317
+ export async function finetune(request, onProgress, context) {
286
318
  if (request.operation === undefined ||
287
319
  request.operation === 'start' ||
288
320
  request.operation === 'resume') {
289
- return startFinetune(request, onProgress);
321
+ return startFinetune(request, onProgress, context);
290
322
  }
323
+ // Under a plugin invoke the caller is the invoke's session; the nested
324
+ // params name whatever the caller put there.
325
+ const callerSessionId = resolveSessionId(context ?? request);
291
326
  switch (request.operation) {
292
327
  case 'getState':
293
328
  return getFinetuneState(request);
294
329
  case 'pause':
295
- return pauseFinetune(request.modelId);
330
+ return pauseFinetune(request.modelId, callerSessionId);
296
331
  case 'cancel':
297
- return cancelFinetune(request.modelId);
332
+ return cancelFinetune(request.modelId, callerSessionId);
298
333
  }
299
334
  }